From 2800e5232e633d672c328e3d9ebbe6e1ea0d22aa Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 01:15:27 +0100 Subject: [PATCH 01/40] feat(placement): add the measured-only planner core and its cost report Add sie_placement.planner.plan, a pure function from service levels, receipts, boot evidence, a price list, demand, the current plan and planner parameters to a plan and a report: - refuse receipts and boot evidence whose realized GPU product differs from the product their machine profile requests or whose profile has no price, and receipts whose latency cannot answer the identity's service level - block cells that miss the warm bounds; offer pinned, and cold-with-bound or snapshot-cold where boot evidence exists, with the slowest successful boot standing for the come-alive percentile, the environment's own evidence replacing borrowed evidence, and a comeAliveP95S of 0 admitting pinned only - price at list times the region multiplier: one warm container plus the above-floor hours demand needs, or the work plus a burst tail held at the active floor for the cooldown; the fallback floor, cooldown, ceiling and buffer are parameters - set the envelope floorMax to floorMin plus the declared buffer while queue wait is unmeasured, at realizable floors and capped by the reviewed ceiling - report per identity the chosen cell, warmth, monthly cost and come-alive basis, and the nearest cheaper blocked option with the bounds that block it; per lane the floor and envelope-maximum cost; measurement requests ranked by their price-bounded saving; come-alive levers priced under the identity's bound and at the bound their measured come-alive permits; unattained queue-wait bounds; refusals; and the changes against the current plan Boot evidence gains an optional borrowed flag. The rendered plan and report are byte-stable under input reordering, and replanning over the plan the planner wrote reports no change. Each identity gets its own lane; co-packing is not part of this change. --- packages/sie_placement/README.md | 116 ++- .../src/sie_placement/planner.py | 763 +++++++++++++++ .../sie_placement/src/sie_placement/report.py | 318 +++++++ .../sie_placement/src/sie_placement/schema.py | 36 +- .../tests/fixtures/planner/boot-evidence.json | 213 +++++ .../tests/fixtures/planner/current-plan.json | 64 ++ .../tests/fixtures/planner/demand.json | 114 +++ .../tests/fixtures/planner/expected-plan.json | 134 +++ .../fixtures/planner/expected-report.json | 698 ++++++++++++++ .../tests/fixtures/planner/price-list.json | 45 + .../tests/fixtures/planner/receipts.json | 295 ++++++ .../fixtures/planner/service-levels.json | 48 + packages/sie_placement/tests/test_levers.py | 215 +++++ packages/sie_placement/tests/test_planner.py | 893 ++++++++++++++++++ packages/sie_placement/tests/test_schema.py | 20 + 15 files changed, 3960 insertions(+), 12 deletions(-) create mode 100644 packages/sie_placement/src/sie_placement/planner.py create mode 100644 packages/sie_placement/src/sie_placement/report.py create mode 100644 packages/sie_placement/tests/fixtures/planner/boot-evidence.json create mode 100644 packages/sie_placement/tests/fixtures/planner/current-plan.json create mode 100644 packages/sie_placement/tests/fixtures/planner/demand.json create mode 100644 packages/sie_placement/tests/fixtures/planner/expected-plan.json create mode 100644 packages/sie_placement/tests/fixtures/planner/expected-report.json create mode 100644 packages/sie_placement/tests/fixtures/planner/price-list.json create mode 100644 packages/sie_placement/tests/fixtures/planner/receipts.json create mode 100644 packages/sie_placement/tests/fixtures/planner/service-levels.json create mode 100644 packages/sie_placement/tests/test_levers.py create mode 100644 packages/sie_placement/tests/test_planner.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 2e434f900..643684cc7 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -3,7 +3,8 @@ Versioned documents for a measured-only placement planner: the service level each served identity must meet, the measurements a placement may be chosen from, the prices those placements cost, the demand they serve, and the plan -that results. The package uses only the Python standard library. It is a +that results, together with the planner that turns them into a plan and a cost +report. The package uses only the Python standard library. It is a workspace package and is not published to PyPI. A served identity is a model id or a `model:profile` pair, the routing key a @@ -74,6 +75,10 @@ A plan gives every identity one warmth: `pinned`, `cold-with-bound`, or more; every other warmth sits on a lane whose `floorMin` is 0. Boot evidence records `cold-with-bound` boots, `snapshot-cold` restores and `lazy-resident` loads, with outcome `succeeded`, `failed` or `timed-out`. +`borrowed: true` marks a boot measured in another environment whose +boot-shaping inputs match: the identity and its model revision, the image, the +realized GPU product and count, the container shape, the warmth and the +placement class. Whoever supplies the evidence decides the match. ## Lanes @@ -94,6 +99,115 @@ lane's `maxContainers`. `bufferMin` with `bufferMax`, and `scaledownWindowMinS` with `scaledownWindowMaxS`, are optional pairs for platforms that can move a warm buffer or a scale-down window per lane. +## Planner + +```python +from sie_placement.planner import PlannerParameters, plan +from sie_placement.report import render + +result = plan( + service_levels, + receipts, + boot_evidence, + price_list, + demand, + current_plan, + PlannerParameters(placement="us-central1", max_containers=4, fallback_floor=2, cooldown_s=900), +) +print(render(result.plan.to_dict()), render(result.report.to_dict())) +``` + +`plan` is a pure function of its documents. The same inputs render to the +same bytes in whatever order the receipts, boot evidence, identities and demand +series arrive, and a run whose `current_plan` is its own previous plan reports +no change. + +| Parameter | Meaning | +| --- | --- | +| `placement` | The price-list placement whose region multiplier applies | +| `max_containers` | The reviewed ceiling of every lane, its `maxContainers` | +| `fallback_floor` | The least containers an active zero-floor lane holds, the `sie-cluster` chart's `autoscaling.fallbackReplicas` | +| `cooldown_s` | How long an active lane keeps its containers after its last request, the chart's `autoscaling.cooldownPeriod` | +| `buffer_containers` | The declared warm buffer above a lane's floor, `0` by default | +| `hours_per_month` | The hours a monthly figure covers, `730` by default | +| `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | + +### Selection + +The planner places each identity of the service-level document on its own +lane: + +1. It refuses a receipt or boot record whose machine profile has no price, or + whose realized GPU product differs from the product the price list gives + that profile, and a receipt whose latency cannot answer the identity's + service level. The report lists every refusal. +2. Every other receipt is a cell. A cell whose p99 misses `warmP99Ms`, or + `ttftP99Ms` or `itlP99Ms`, is blocked. +3. A cell offers `pinned`, and `cold-with-bound` or `snapshot-cold` where boot + evidence for the identity, cell and container shape exists. The slowest + successful boot stands for the come-alive percentile, a boot that did not + succeed withdraws the warmth, and the environment's own evidence replaces + borrowed evidence. A `comeAliveP95S` of `0` admits `pinned` only. +4. The cheapest unblocked option wins. Ties go to the current plan's + placement, then to the faster come-alive. + +The planner does not select `lazy-resident`, `bridged` or `scheduled-warm`, +and does not put two identities on one lane. An identity with no unblocked +option keeps its current lane, or stays out of the plan when it has none. + +### Pricing + +A container costs `(acceleratorPerHour * gpuCount + corePerHour * cpuCores + +memoryGiBPerHour * memoryMiB / 1024) * regionMultiplier` an hour. An identity's +demand is its hourly requests summed over operations. A container serves +`requestsPerSecond * 3600` requests an hour at the receipt's concurrency, and +no hour runs more containers than the ceiling. Monthly figures scale the +demand window to `hours_per_month`. + +- `pinned` pays one container all month, plus each container an hour needs + above it, for that hour. +- A cold warmth pays, in each hour with requests, the work plus a tail after + every request: the active floor for `cooldown_s`, at most the whole hour. + The active floor is `fallback_floor` capped by the ceiling, and at least one. + +### Envelope + +`floorMin` is 1 for `pinned` and 0 otherwise. No source measures queue wait +yet, so `floorMax` is `floorMin` plus `buffer_containers`, capped by the +ceiling. A floor raised on a zero-floor lane realizes as at least +`fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least +`fallback_floor` capped by the ceiling. The envelope carries the floor range +only. + +### Report + +The report renders as a `placement-report` document: + +- `identities`: each identity's status (`placed`, `kept` or `unplaced`), + effective service level and demand, the chosen cell, warmth, monthly cost and + come-alive with its basis (`warm-floor`, `boot-evidence` or + `borrowed-boot-evidence`), the peak containers its demand needs, and + `nextCheaper`: the nearest cheaper option a service level blocks, its saving + and the bounds that block it. +- `lanes`: each lane's hourly cost, its floor's monthly cost and the monthly + cost of its `floorMax`, which a floor budget prices. +- `measurementRequests`: options that lack evidence, ranked by the saving they + could unlock. A profile without a receipt is bounded by price alone: one + accelerator at its rate, kept warm or, where the come-alive bound allows, + paying burst tails only. A cell whose receipt has no successful boot asks for + boot evidence and is bounded by its cold cost. A request that cannot save is + dropped, and the requests of an identity without a placement carry no bound + and rank first. +- `comeAliveLevers`: cold warmths whose measured come-alive misses the bound + on a cell that meets the warm bounds for less than the choice. Each is priced + under the identity's bound, where a bound of 0 admits no lever, and at + `permittedBoundS`, the bound its measured come-alive would permit. Levers rank + by the first saving, then by the second. +- `queueWaitUnattained`: every identity that declares `queueWaitP99Ms`. No + plan claims that bound as met. +- `refusals`, and `changes`: the identities and lanes that differ from + `current_plan`. + ## Tests ```bash diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py new file mode 100644 index 000000000..29b23f997 --- /dev/null +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -0,0 +1,763 @@ +"""Measured-only placement: the cheapest measured cell and warmth that meet each identity's service level.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime + +from sie_placement.report import ( + Blocker, + ComeAliveBasis, + IdentityChange, + IdentityReport, + IdentityStatus, + LaneChange, + LaneReport, + Lever, + Measurement, + MeasurementRequest, + Option, + Refusal, + RefusedRecord, + Report, + RequestReason, + UnplacedReason, +) +from sie_placement.schema import ( + WARM_FLOOR_WARMTHS, + BootEvidence, + BootOutcome, + Cell, + ContainerShape, + Demand, + Envelope, + MachineProfilePrice, + Operation, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + SchemaError, + ServiceLevel, + ServiceLevels, + SourceRef, + Warmth, +) + +SECONDS_PER_HOUR = 3600 +HOURS_PER_MONTH = 730.0 +PINNED_FLOOR = 1 +COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) + +_SLUG = re.compile(r"[^a-z0-9]+") + + +class PlannerInputError(ValueError): + pass + + +def _require_name(name: str, value: str) -> None: + if not value or any(character.isspace() for character in value): + raise SchemaError(name, f"must be a non-empty string without whitespace, got {value!r}") + + +def _require_minimum(name: str, value: float, minimum: int) -> None: + if value < minimum: + raise SchemaError(name, f"must be at least {minimum}, got {value!r}") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class PlannerParameters: + placement: str + max_containers: int + fallback_floor: int + cooldown_s: int + buffer_containers: int = 0 + hours_per_month: float = HOURS_PER_MONTH + queue_pool: str = "default" + bundle: str = "default" + bundles: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + _require_name("placement", self.placement) + _require_minimum("maxContainers", self.max_containers, 1) + _require_minimum("fallbackFloor", self.fallback_floor, 0) + _require_minimum("cooldownS", self.cooldown_s, 0) + _require_minimum("bufferContainers", self.buffer_containers, 0) + if not (math.isfinite(self.hours_per_month) and self.hours_per_month > 0): + raise SchemaError("hoursPerMonth", f"must be greater than 0, got {self.hours_per_month!r}") + _require_name("queuePool", self.queue_pool) + _require_name("bundle", self.bundle) + for identity, bundle in self.bundles.items(): + _require_name("bundles", identity) + _require_name(f"bundles[{identity!r}]", bundle) + + @property + def active_floor(self) -> int: + return max(1, min(self.fallback_floor, self.max_containers)) + + def bundle_of(self, identity: str) -> str: + return self.bundles.get(identity, self.bundle) + + def to_dict(self) -> dict[str, object]: + document: dict[str, object] = { + "placement": self.placement, + "maxContainers": self.max_containers, + "fallbackFloor": self.fallback_floor, + "cooldownS": self.cooldown_s, + "bufferContainers": self.buffer_containers, + "hoursPerMonth": self.hours_per_month, + "queuePool": self.queue_pool, + "bundle": self.bundle, + } + if self.bundles: + document["bundles"] = dict(sorted(self.bundles.items())) + return document + + +@dataclass(frozen=True, slots=True, kw_only=True) +class PlannerResult: + plan: Plan + report: Report + + +def container_hourly_cost(price: MachineProfilePrice, shape: ContainerShape, region_multiplier: float) -> float: + return region_multiplier * ( + price.accelerator_per_hour * shape.gpu_count + + price.core_per_hour * shape.cpu_cores + + price.memory_gib_per_hour * shape.memory_mib / 1024 + ) + + +def envelope_for(warmth: Warmth, parameters: PlannerParameters) -> Envelope: + floor_min = PINNED_FLOOR if warmth in WARM_FLOOR_WARMTHS else 0 + raised = floor_min + parameters.buffer_containers + if raised and not floor_min: + raised = max(raised, parameters.fallback_floor) + return Envelope(floor_min=floor_min, floor_max=min(parameters.max_containers, raised)) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Window: + hours: int + hours_per_month: float + ceiling: int + active_floor: int + cooldown_s: int + + @property + def scale(self) -> float: + return self.hours_per_month / self.hours + + @staticmethod + def needed(requests: int, requests_per_second: float) -> int: + return math.ceil(requests / (requests_per_second * SECONDS_PER_HOUR)) + + def warm_container_hours(self, requests: Sequence[int], requests_per_second: float) -> float: + above = sum( + max(0, min(self.ceiling, self.needed(count, requests_per_second)) - PINNED_FLOOR) for count in requests + ) + return PINNED_FLOOR * self.hours_per_month + above * self.scale + + def cold_container_hours(self, requests: Sequence[int], requests_per_second: float | None) -> float: + seconds = 0.0 + for count in requests: + tail = count * self.active_floor * self.cooldown_s + if requests_per_second is None: + seconds += min(self.active_floor * SECONDS_PER_HOUR, tail) + continue + containers = max(self.active_floor, min(self.ceiling, self.needed(count, requests_per_second))) + seconds += min(containers * SECONDS_PER_HOUR, count / requests_per_second + tail) + return seconds / SECONDS_PER_HOUR * self.scale + + +@dataclass(frozen=True, slots=True) +class _Boot: + seconds: float | None + borrowed: bool + + @property + def basis(self) -> ComeAliveBasis: + return ComeAliveBasis.BORROWED_BOOT_EVIDENCE if self.borrowed else ComeAliveBasis.BOOT_EVIDENCE + + +type _BootKey = tuple[str, Cell, ContainerShape, Warmth] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Option: + receipt: Receipt + order: str + warmth: Warmth + cost: float + come_alive_s: float + basis: ComeAliveBasis + blockers: tuple[Blocker, ...] + current: bool + + @property + def feasible(self) -> bool: + return not self.blockers + + @property + def rank(self) -> tuple[float, bool, float, str, str]: + return (self.cost, not self.current, self.come_alive_s, self.order, self.warmth.value) + + def summary(self, saving: float | None = None) -> Option: + return Option( + cell=self.receipt.cell, + container_shape=self.receipt.container_shape, + concurrency=self.receipt.concurrency, + warmth=self.warmth, + monthly_cost=self.cost, + come_alive_s=self.come_alive_s, + come_alive_basis=self.basis, + receipt=self.receipt.source, + blocked_by=self.blockers, + saving=saving, + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Choice: + identity: str + level: ServiceLevel + requests: tuple[int, ...] + options: tuple[_Option, ...] + chosen: _Option | None + boot_requests: tuple[MeasurementRequest, ...] + has_valid_receipt: bool + + +def _canonical(document: Mapping[str, object]) -> str: + return json.dumps(document, sort_keys=True) + + +def _slug(value: str) -> str: + return _SLUG.sub("-", value.lower()).strip("-") + + +def _digest(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest()[:8] + + +def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> str | None: + price = prices.get(cell.machine_profile) + if price is None: + return f"machine profile {cell.machine_profile!r} has no price" + if price.gpu_product != cell.gpu_product: + return ( + f"realized GPU product {cell.gpu_product!r} differs from {price.gpu_product!r}, " + f"the product machine profile {cell.machine_profile!r} requests" + ) + return None + + +def _incomparable_reason(level: ServiceLevel, receipt: Receipt) -> str | None: + generate = receipt.request_shape.operation is Operation.GENERATE + if level.is_generation and not generate: + return "the receipt reports one latency and the service level declares ttftP99Ms and itlP99Ms" + if level.warm_p99_ms is not None and generate: + return "the receipt reports ttft and itl and the service level declares warmP99Ms" + return None + + +def _warm_blockers(level: ServiceLevel, receipt: Receipt) -> tuple[Blocker, ...]: + checks: list[tuple[str, int | None, float]] = [] + if receipt.latency is not None: + checks.append(("warmP99Ms", level.warm_p99_ms, receipt.latency.p99_ms)) + if receipt.ttft is not None: + checks.append(("ttftP99Ms", level.ttft_p99_ms, receipt.ttft.p99_ms)) + if receipt.itl is not None: + checks.append(("itlP99Ms", level.itl_p99_ms, receipt.itl.p99_ms)) + return tuple( + Blocker(field=name, bound=bound, measured=measured) + for name, bound, measured in checks + if bound is not None and measured > bound + ) + + +def _come_alive_blocker(level: ServiceLevel, seconds: float) -> Blocker | None: + bound = level.come_alive_p95_s + if bound is None or (bound > 0 and seconds <= bound): + return None + return Blocker(field="comeAliveP95S", bound=bound, measured=seconds) + + +def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: + grouped: defaultdict[_BootKey, list[BootEvidence]] = defaultdict(list) + for record in records: + grouped[(record.identity, record.cell, record.container_shape, record.warmth)].append(record) + index: dict[_BootKey, _Boot] = {} + for key, group in grouped.items(): + own = [record for record in group if not record.borrowed] + used = own or group + if any(record.outcome is not BootOutcome.SUCCEEDED for record in used): + index[key] = _Boot(seconds=None, borrowed=not own) + else: + index[key] = _Boot(seconds=max(record.boot_seconds for record in used), borrowed=not own) + return index + + +def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: + hours: defaultdict[str, Counter[datetime]] = defaultdict(Counter) + for series in demand.series: + for hour in series.hours: + hours[series.identity][hour.start] += hour.requests + return { + identity: tuple(count for _, count in sorted(counts.items()) if count) for identity, counts in hours.items() + } + + +def _matches_current(receipt: Receipt, warmth: Warmth, current: tuple[PlanIdentity, PlanLane] | None) -> bool: + if current is None: + return False + entry, lane = current + return ( + entry.warmth is warmth + and lane.cell == receipt.cell + and lane.container_shape == receipt.container_shape + and lane.concurrency == receipt.concurrency + ) + + +class _Planner: + def __init__( + self, + service_levels: ServiceLevels, + receipts: Iterable[Receipt], + boot_evidence: Iterable[BootEvidence], + price_list: PriceList, + demand: Demand, + current_plan: Plan | None, + parameters: PlannerParameters, + ) -> None: + placements = {placement.name: placement for placement in price_list.placements} + placement = placements.get(parameters.placement) + if placement is None: + raise PlannerInputError(f"placement {parameters.placement!r} is not in the price list") + if current_plan is not None and current_plan.placement != parameters.placement: + raise PlannerInputError( + f"the current plan is for placement {current_plan.placement!r}, not {parameters.placement!r}" + ) + self.parameters = parameters + self.price_list = price_list + self.multiplier = placement.region_multiplier + self.prices = {profile.name: profile for profile in price_list.machine_profiles} + self.levels = {entry.identity: service_levels.effective(entry.identity) for entry in service_levels.identities} + self.window = _Window( + hours=int((demand.end - demand.start).total_seconds()) // SECONDS_PER_HOUR, + hours_per_month=parameters.hours_per_month, + ceiling=parameters.max_containers, + active_floor=parameters.active_floor, + cooldown_s=parameters.cooldown_s, + ) + self.requests = _hourly_requests(demand) + self.current_plan = current_plan + self.current: dict[str, tuple[PlanIdentity, PlanLane]] = {} + if current_plan is not None: + lanes = {lane.name: lane for lane in current_plan.lanes} + self.current = {entry.identity: (entry, lanes[entry.lane]) for entry in current_plan.identities} + self.refusals: list[Refusal] = [] + self.refused_profiles: set[tuple[str, str]] = set() + self.receipts = self._accept_receipts(receipts) + self.boots = _boot_index(self._accept_boot_evidence(boot_evidence)) + + def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[str, Receipt]]]: + accepted: defaultdict[str, list[tuple[str, Receipt]]] = defaultdict(list) + for receipt in receipts: + level = self.levels.get(receipt.identity) + if level is None: + continue + reason = _refusal_reason(receipt.cell, self.prices) or _incomparable_reason(level, receipt) + if reason is not None: + self._refuse(RefusedRecord.RECEIPT, receipt.identity, receipt.cell, receipt.source, reason) + self.refused_profiles.add((receipt.identity, receipt.cell.machine_profile)) + continue + accepted[receipt.identity].append((_canonical(receipt.to_dict()), receipt)) + return accepted + + def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvidence]: + accepted = [] + for record in records: + if record.identity not in self.levels: + continue + reason = _refusal_reason(record.cell, self.prices) + if reason is not None: + self._refuse(RefusedRecord.BOOT_EVIDENCE, record.identity, record.cell, record.source, reason) + continue + accepted.append(record) + return accepted + + def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: + self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) + + def hourly_cost(self, cell: Cell, shape: ContainerShape) -> float | None: + if _refusal_reason(cell, self.prices) is not None: + return None + return container_hourly_cost(self.prices[cell.machine_profile], shape, self.multiplier) + + def choose(self, identity: str) -> _Choice: + level = self.levels[identity] + requests = self.requests.get(identity, ()) + current = self.current.get(identity) + options: list[_Option] = [] + boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} + receipts = self.receipts.get(identity, []) + for order, receipt in receipts: + hourly = container_hourly_cost( + self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier + ) + rate = receipt.throughput.requests_per_second + warm = _warm_blockers(level, receipt) + options.append( + _Option( + receipt=receipt, + order=order, + warmth=Warmth.PINNED, + cost=hourly * self.window.warm_container_hours(requests, rate), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=warm, + current=_matches_current(receipt, Warmth.PINNED, current), + ) + ) + cold_cost = hourly * self.window.cold_container_hours(requests, rate) + for warmth in COLD_WARMTHS: + boot = self.boots.get((identity, receipt.cell, receipt.container_shape, warmth)) + if boot is None or boot.seconds is None: + if warmth is Warmth.COLD_WITH_BOUND and not warm and level.come_alive_p95_s != 0: + key = (receipt.cell, receipt.container_shape) + known = boot_requests.get(key) + if known is None or cold_cost < known.cost_bound: + boot_requests[key] = MeasurementRequest( + identity=identity, + cell=receipt.cell, + container_shape=receipt.container_shape, + measure=(Measurement.BOOT_EVIDENCE,), + warmth=warmth, + reason=RequestReason.UNMEASURED if boot is None else RequestReason.FAILED, + cost_bound=cold_cost, + saving_bound=None, + ) + continue + blocker = _come_alive_blocker(level, boot.seconds) + options.append( + _Option( + receipt=receipt, + order=order, + warmth=warmth, + cost=cold_cost, + come_alive_s=boot.seconds, + basis=boot.basis, + blockers=warm if blocker is None else (*warm, blocker), + current=_matches_current(receipt, warmth, current), + ) + ) + feasible = [option for option in options if option.feasible] + chosen = min(feasible, key=lambda option: option.rank) if feasible else None + return _Choice( + identity=identity, + level=level, + requests=requests, + options=tuple(options), + chosen=chosen, + boot_requests=tuple(boot_requests.values()), + has_valid_receipt=bool(receipts), + ) + + def receipt_requests(self, choice: _Choice) -> list[MeasurementRequest]: + measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} + bound = choice.level.come_alive_p95_s + requests = [] + for profile in sorted(self.price_list.machine_profiles, key=lambda price: price.name): + if profile.name in measured: + continue + accelerator = profile.accelerator_per_hour * self.multiplier + candidates = [(accelerator * self.window.warm_container_hours((), 1.0), Warmth.PINNED)] + if bound != 0: + cold = accelerator * self.window.cold_container_hours(choice.requests, None) + candidates.append((cold, Warmth.COLD_WITH_BOUND)) + cost, warmth = min(candidates, key=lambda candidate: (candidate[0], candidate[1] is not Warmth.PINNED)) + saving = None if choice.chosen is None else choice.chosen.cost - cost + if saving is not None and saving <= 0: + continue + requests.append( + MeasurementRequest( + identity=choice.identity, + cell=Cell(machine_profile=profile.name, gpu_product=profile.gpu_product), + measure=( + (Measurement.RECEIPT,) + if warmth is Warmth.PINNED + else (Measurement.RECEIPT, Measurement.BOOT_EVIDENCE) + ), + warmth=warmth, + reason=( + RequestReason.REFUSED + if (choice.identity, profile.name) in self.refused_profiles + else RequestReason.UNMEASURED + ), + cost_bound=cost, + saving_bound=saving, + ) + ) + return requests + + def boot_requests(self, choice: _Choice) -> list[MeasurementRequest]: + if choice.chosen is None: + return [] + chosen = choice.chosen + return [ + replace(request, saving_bound=chosen.cost - request.cost_bound) + for request in choice.boot_requests + if chosen.cost - request.cost_bound > 0 + ] + + @staticmethod + def levers(choice: _Choice) -> list[Lever]: + chosen = choice.chosen + bound = choice.level.come_alive_p95_s + if chosen is None or bound is None: + return [] + best: dict[tuple[Cell, ContainerShape, Warmth], _Option] = {} + for option in choice.options: + if option.warmth not in COLD_WARMTHS or option.cost >= chosen.cost: + continue + if [blocker.field for blocker in option.blockers] != ["comeAliveP95S"]: + continue + key = (option.receipt.cell, option.receipt.container_shape, option.warmth) + known = best.get(key) + if known is None or option.rank < known.rank: + best[key] = option + levers = [] + for option in best.values(): + saving = chosen.cost - option.cost + levers.append( + Lever( + identity=choice.identity, + cell=option.receipt.cell, + container_shape=option.receipt.container_shape, + warmth=option.warmth, + come_alive_s=option.come_alive_s, + come_alive_basis=option.basis, + bound_s=bound, + monthly_cost=option.cost, + saving_under_bound=saving if bound > 0 else 0.0, + permitted_bound_s=max(1, math.ceil(option.come_alive_s)), + saving_at_permitted_bound=saving, + ) + ) + return levers + + @staticmethod + def next_cheaper(choice: _Choice) -> _Option | None: + chosen = choice.chosen + if chosen is None: + return None + cheaper = [option for option in choice.options if not option.feasible and option.cost < chosen.cost] + if not cheaper: + return None + nearest = max(option.cost for option in cheaper) + return min((option for option in cheaper if option.cost == nearest), key=lambda option: option.rank) + + def lane_names(self, choices: Sequence[_Choice], reserved: set[str]) -> dict[str, str]: + bases: dict[str, str] = {} + for choice in choices: + if choice.chosen is not None: + profile = choice.chosen.receipt.cell.machine_profile + bases[choice.identity] = _slug(f"{choice.identity}-{profile}") or "lane" + counts = Counter(bases.values()) + return { + identity: base if counts[base] == 1 and base not in reserved else f"{base}-{_digest(identity)}" + for identity, base in bases.items() + } + + def run(self) -> PlannerResult: + choices = [self.choose(identity) for identity in sorted(self.levels)] + kept = { + choice.identity: self.current[choice.identity] + for choice in choices + if choice.chosen is None and choice.identity in self.current + } + kept_lanes = {lane.name: lane for _, lane in kept.values()} + names = self.lane_names(choices, set(kept_lanes)) + lanes: dict[str, PlanLane] = dict(kept_lanes) + identities: list[PlanIdentity] = [entry for entry, _ in kept.values()] + reports: list[IdentityReport] = [] + requests: list[MeasurementRequest] = [] + levers: list[Lever] = [] + monthly_cost = 0.0 + for choice in choices: + requests.extend(self.receipt_requests(choice)) + requests.extend(self.boot_requests(choice)) + levers.extend(self.levers(choice)) + reports.append(self._identity_report(choice, names, kept)) + if choice.chosen is None: + continue + monthly_cost += choice.chosen.cost + receipt = choice.chosen.receipt + name = names[choice.identity] + lanes[name] = PlanLane( + name=name, + queue_pool=self.parameters.queue_pool, + bundle=self.parameters.bundle_of(choice.identity), + cell=receipt.cell, + container_shape=receipt.container_shape, + concurrency=receipt.concurrency, + max_containers=self.parameters.max_containers, + envelope=envelope_for(choice.chosen.warmth, self.parameters), + ) + identities.append(PlanIdentity(identity=choice.identity, lane=name, warmth=choice.chosen.warmth)) + new_plan = Plan( + placement=self.parameters.placement, + lanes=tuple(lanes[name] for name in sorted(lanes)), + identities=tuple(sorted(identities, key=lambda entry: entry.identity)), + ) + lane_reports = self._lane_reports(new_plan) + identity_changes, lane_changes = _changes(self.current_plan, new_plan) + report = Report( + placement=self.parameters.placement, + currency=self.price_list.currency, + parameters=self.parameters.to_dict(), + monthly_cost=monthly_cost, + envelope_max_monthly_cost=sum( + lane.envelope_max_monthly_cost for lane in lane_reports if lane.envelope_max_monthly_cost is not None + ), + identities=tuple(reports), + lanes=lane_reports, + measurement_requests=tuple(sorted(requests, key=_request_rank)), + come_alive_levers=tuple(sorted(levers, key=_lever_rank)), + queue_wait_unattained=tuple( + identity for identity in sorted(self.levels) if self.levels[identity].queue_wait_p99_ms is not None + ), + refusals=tuple(sorted(self.refusals, key=_refusal_rank)), + identity_changes=identity_changes, + lane_changes=lane_changes, + ) + return PlannerResult(plan=new_plan, report=report) + + def _identity_report( + self, choice: _Choice, names: Mapping[str, str], kept: Mapping[str, tuple[PlanIdentity, PlanLane]] + ) -> IdentityReport: + current = self.current.get(choice.identity) + report = IdentityReport( + identity=choice.identity, + status=IdentityStatus.UNPLACED, + service_level=choice.level, + demand_requests=sum(choice.requests), + current=None if current is None else current[0], + ) + chosen = choice.chosen + if chosen is None: + reason = UnplacedReason.WARM_BOUND_MISSED if choice.has_valid_receipt else UnplacedReason.NO_RECEIPT + if choice.identity in kept: + return replace( + report, status=IdentityStatus.KEPT, lane=kept[choice.identity][0].lane, unplaced_reason=reason + ) + return replace(report, unplaced_reason=reason) + nearest = self.next_cheaper(choice) + rate = chosen.receipt.throughput.requests_per_second + return replace( + report, + status=IdentityStatus.PLACED, + lane=names[choice.identity], + chosen=chosen.summary(), + peak_containers=max((_Window.needed(count, rate) for count in choice.requests), default=0), + next_cheaper=None if nearest is None else nearest.summary(saving=chosen.cost - nearest.cost), + ) + + def _lane_reports(self, plan: Plan) -> tuple[LaneReport, ...]: + residents: defaultdict[str, list[str]] = defaultdict(list) + for entry in plan.identities: + residents[entry.lane].append(entry.identity) + reports = [] + for lane in plan.lanes: + hourly = self.hourly_cost(lane.cell, lane.container_shape) + hours = self.parameters.hours_per_month + reports.append( + LaneReport( + name=lane.name, + identities=tuple(sorted(residents[lane.name])), + hourly_cost=hourly, + floor_monthly_cost=None if hourly is None else hourly * lane.envelope.floor_min * hours, + envelope_max_monthly_cost=None if hourly is None else hourly * lane.envelope.floor_max * hours, + ) + ) + return tuple(reports) + + +def _changes(current: Plan | None, new: Plan) -> tuple[tuple[IdentityChange, ...], tuple[LaneChange, ...]]: + before_identities = {} if current is None else {entry.identity: entry for entry in current.identities} + after_identities = {entry.identity: entry for entry in new.identities} + before_lanes = {} if current is None else {lane.name: lane for lane in current.lanes} + after_lanes = {lane.name: lane for lane in new.lanes} + identity_changes = tuple( + IdentityChange(identity=identity, before=before_identities.get(identity), after=after_identities.get(identity)) + for identity in sorted(before_identities.keys() | after_identities.keys()) + if before_identities.get(identity) != after_identities.get(identity) + ) + lane_changes = tuple( + LaneChange(name=name, before=before_lanes.get(name), after=after_lanes.get(name)) + for name in sorted(before_lanes.keys() | after_lanes.keys()) + if before_lanes.get(name) != after_lanes.get(name) + ) + return identity_changes, lane_changes + + +def _shape_key(shape: ContainerShape | None) -> tuple[int, float, int]: + return (0, 0.0, 0) if shape is None else (shape.gpu_count, shape.cpu_cores, shape.memory_mib) + + +def _request_rank(request: MeasurementRequest) -> tuple[object, ...]: + return ( + request.saving_bound is not None, + -(request.saving_bound or 0.0), + request.identity, + request.cell.machine_profile, + request.cell.gpu_product, + _shape_key(request.container_shape), + tuple(measurement.value for measurement in request.measure), + ) + + +def _lever_rank(lever: Lever) -> tuple[object, ...]: + return ( + -lever.saving_under_bound, + -lever.saving_at_permitted_bound, + lever.identity, + lever.cell.machine_profile, + lever.cell.gpu_product, + _shape_key(lever.container_shape), + lever.warmth.value, + ) + + +def _refusal_rank(refusal: Refusal) -> tuple[str, ...]: + return ( + refusal.record.value, + refusal.identity, + refusal.cell.machine_profile, + refusal.cell.gpu_product, + refusal.source.uri, + refusal.source.sha256 or "", + refusal.reason, + ) + + +def plan( + service_levels: ServiceLevels, + receipts: Iterable[Receipt], + boot_evidence: Iterable[BootEvidence], + price_list: PriceList, + demand: Demand, + current_plan: Plan | None, + parameters: PlannerParameters, +) -> PlannerResult: + return _Planner(service_levels, receipts, boot_evidence, price_list, demand, current_plan, parameters).run() diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py new file mode 100644 index 000000000..462bbe3ed --- /dev/null +++ b/packages/sie_placement/src/sie_placement/report.py @@ -0,0 +1,318 @@ +"""The planner's report: what each placement costs and what would make it cheaper.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum + +from sie_placement.schema import ( + SCHEMA_VERSION, + Cell, + ContainerShape, + PlanIdentity, + PlanLane, + ServiceLevel, + SourceRef, + Warmth, +) + +REPORT_KIND = "placement-report" + + +class IdentityStatus(StrEnum): + PLACED = "placed" + KEPT = "kept" + UNPLACED = "unplaced" + + +class UnplacedReason(StrEnum): + NO_RECEIPT = "no-receipt" + WARM_BOUND_MISSED = "warm-bound-missed" + + +class ComeAliveBasis(StrEnum): + WARM_FLOOR = "warm-floor" + BOOT_EVIDENCE = "boot-evidence" + BORROWED_BOOT_EVIDENCE = "borrowed-boot-evidence" + + +class Measurement(StrEnum): + RECEIPT = "receipt" + BOOT_EVIDENCE = "boot-evidence" + + +class RequestReason(StrEnum): + UNMEASURED = "unmeasured" + REFUSED = "refused" + FAILED = "failed" + + +class RefusedRecord(StrEnum): + RECEIPT = "receipt" + BOOT_EVIDENCE = "boot-evidence" + + +def _money(value: float) -> float: + return round(value, 2) + 0.0 + + +def _rate(value: float) -> float: + return round(value, 4) + 0.0 + + +def _compact(fields: dict[str, object | None]) -> dict[str, object]: + return {key: value for key, value in fields.items() if value is not None} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Blocker: + field: str + bound: int + measured: float + + def to_dict(self) -> dict[str, object]: + return {"field": self.field, "bound": self.bound, "measured": self.measured} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Option: + cell: Cell + container_shape: ContainerShape + concurrency: int + warmth: Warmth + monthly_cost: float + come_alive_s: float + come_alive_basis: ComeAliveBasis + receipt: SourceRef + blocked_by: tuple[Blocker, ...] = () + saving: float | None = None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "concurrency": self.concurrency, + "warmth": self.warmth.value, + "monthlyCost": _money(self.monthly_cost), + "saving": None if self.saving is None else _money(self.saving), + "comeAliveS": self.come_alive_s, + "comeAliveBasis": self.come_alive_basis.value, + "blockedBy": [blocker.to_dict() for blocker in self.blocked_by] or None, + "receipt": self.receipt.to_dict(), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class IdentityReport: + identity: str + status: IdentityStatus + service_level: ServiceLevel + demand_requests: int + lane: str | None = None + chosen: Option | None = None + peak_containers: int | None = None + next_cheaper: Option | None = None + current: PlanIdentity | None = None + unplaced_reason: UnplacedReason | None = None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "identity": self.identity, + "status": self.status.value, + "serviceLevel": self.service_level.to_dict(), + "demandRequests": self.demand_requests, + "lane": self.lane, + "chosen": None if self.chosen is None else self.chosen.to_dict(), + "peakContainers": self.peak_containers, + "nextCheaper": None if self.next_cheaper is None else self.next_cheaper.to_dict(), + "unplacedReason": None if self.unplaced_reason is None else self.unplaced_reason.value, + "current": None if self.current is None else self.current.to_dict(), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class LaneReport: + name: str + identities: tuple[str, ...] + hourly_cost: float | None + floor_monthly_cost: float | None + envelope_max_monthly_cost: float | None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "name": self.name, + "identities": list(self.identities), + "hourlyCost": None if self.hourly_cost is None else _rate(self.hourly_cost), + "floorMonthlyCost": None if self.floor_monthly_cost is None else _money(self.floor_monthly_cost), + "envelopeMaxMonthlyCost": ( + None if self.envelope_max_monthly_cost is None else _money(self.envelope_max_monthly_cost) + ), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class MeasurementRequest: + identity: str + cell: Cell + measure: tuple[Measurement, ...] + warmth: Warmth + reason: RequestReason + cost_bound: float + saving_bound: float | None + container_shape: ContainerShape | None = None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "identity": self.identity, + "cell": self.cell.to_dict(), + "containerShape": None if self.container_shape is None else self.container_shape.to_dict(), + "measure": [measurement.value for measurement in self.measure], + "warmth": self.warmth.value, + "reason": self.reason.value, + "costBound": _money(self.cost_bound), + "savingBound": None if self.saving_bound is None else _money(self.saving_bound), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Lever: + identity: str + cell: Cell + container_shape: ContainerShape + warmth: Warmth + come_alive_s: float + come_alive_basis: ComeAliveBasis + bound_s: int + monthly_cost: float + saving_under_bound: float + permitted_bound_s: int + saving_at_permitted_bound: float + + def to_dict(self) -> dict[str, object]: + return { + "identity": self.identity, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "warmth": self.warmth.value, + "comeAliveS": self.come_alive_s, + "comeAliveBasis": self.come_alive_basis.value, + "comeAliveBoundS": self.bound_s, + "monthlyCost": _money(self.monthly_cost), + "savingUnderBound": _money(self.saving_under_bound), + "permittedBoundS": self.permitted_bound_s, + "savingAtPermittedBound": _money(self.saving_at_permitted_bound), + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Refusal: + record: RefusedRecord + identity: str + cell: Cell + source: SourceRef + reason: str + + def to_dict(self) -> dict[str, object]: + return { + "record": self.record.value, + "identity": self.identity, + "cell": self.cell.to_dict(), + "source": self.source.to_dict(), + "reason": self.reason, + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class IdentityChange: + identity: str + before: PlanIdentity | None + after: PlanIdentity | None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "identity": self.identity, + "before": None if self.before is None else self.before.to_dict(), + "after": None if self.after is None else self.after.to_dict(), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class LaneChange: + name: str + before: PlanLane | None + after: PlanLane | None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "name": self.name, + "before": None if self.before is None else self.before.to_dict(), + "after": None if self.after is None else self.after.to_dict(), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Report: + placement: str + currency: str + parameters: Mapping[str, object] + monthly_cost: float + envelope_max_monthly_cost: float + identities: tuple[IdentityReport, ...] + lanes: tuple[LaneReport, ...] + measurement_requests: tuple[MeasurementRequest, ...] + come_alive_levers: tuple[Lever, ...] + queue_wait_unattained: tuple[str, ...] + refusals: tuple[Refusal, ...] + identity_changes: tuple[IdentityChange, ...] + lane_changes: tuple[LaneChange, ...] + + @property + def unchanged(self) -> bool: + return not self.identity_changes and not self.lane_changes + + def identity(self, identity: str) -> IdentityReport: + for entry in self.identities: + if entry.identity == identity: + return entry + raise KeyError(identity) + + def to_dict(self) -> dict[str, object]: + return { + "kind": REPORT_KIND, + "schemaVersion": SCHEMA_VERSION, + "placement": self.placement, + "currency": self.currency, + "parameters": dict(self.parameters), + "totals": { + "monthlyCost": _money(self.monthly_cost), + "envelopeMaxMonthlyCost": _money(self.envelope_max_monthly_cost), + }, + "identities": [entry.to_dict() for entry in self.identities], + "lanes": [lane.to_dict() for lane in self.lanes], + "measurementRequests": [request.to_dict() for request in self.measurement_requests], + "comeAliveLevers": [lever.to_dict() for lever in self.come_alive_levers], + "queueWaitUnattained": list(self.queue_wait_unattained), + "refusals": [refusal.to_dict() for refusal in self.refusals], + "changes": { + "identities": [change.to_dict() for change in self.identity_changes], + "lanes": [change.to_dict() for change in self.lane_changes], + }, + } + + +def render(document: Mapping[str, object]) -> str: + return json.dumps(document, indent=2, ensure_ascii=False) + "\n" diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index 410d6cf7e..ba9e691ba 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -135,6 +135,11 @@ def _string(self, key: str, value: object) -> str: raise SchemaError(self._at(key), f"expected a string, got {_describe(value)}") return value + def _boolean(self, key: str, value: object) -> bool: + if not isinstance(value, bool): + raise SchemaError(self._at(key), f"expected a boolean, got {_describe(value)}") + return value + def _integer(self, key: str, value: object) -> int: if isinstance(value, bool) or not isinstance(value, int): raise SchemaError(self._at(key), f"expected an integer, got {_describe(value)}") @@ -171,6 +176,10 @@ def optional_string(self, key: str) -> str | None: value = self._optional(key) return None if value is None else self._string(key, value) + def optional_boolean(self, key: str) -> bool | None: + value = self._optional(key) + return None if value is None else self._boolean(key, value) + def integer(self, key: str) -> int: return self._integer(key, self._required(key)) @@ -661,6 +670,7 @@ class BootEvidence: boot_seconds: float outcome: BootOutcome source: SourceRef + borrowed: bool = False def __post_init__(self) -> None: _require_identifier("identity", self.identity) @@ -681,21 +691,25 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: warmth=reader.enum("warmth", Warmth), boot_seconds=reader.number("bootSeconds"), outcome=reader.enum("outcome", BootOutcome), + borrowed=reader.optional_boolean("borrowed") is True, source=reader.child("source", SourceRef.from_dict), ) def to_dict(self) -> dict[str, object]: - return { - "kind": BOOT_EVIDENCE_KIND, - "schemaVersion": SCHEMA_VERSION, - "identity": self.identity, - "cell": self.cell.to_dict(), - "containerShape": self.container_shape.to_dict(), - "warmth": self.warmth.value, - "bootSeconds": self.boot_seconds, - "outcome": self.outcome.value, - "source": self.source.to_dict(), - } + return _compact( + { + "kind": BOOT_EVIDENCE_KIND, + "schemaVersion": SCHEMA_VERSION, + "identity": self.identity, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "warmth": self.warmth.value, + "bootSeconds": self.boot_seconds, + "outcome": self.outcome.value, + "borrowed": True if self.borrowed else None, + "source": self.source.to_dict(), + } + ) @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/sie_placement/tests/fixtures/planner/boot-evidence.json b/packages/sie_placement/tests/fixtures/planner/boot-evidence.json new file mode 100644 index 000000000..4667e5138 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/boot-evidence.json @@ -0,0 +1,213 @@ +[ + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "bootSeconds": 95.0, + "outcome": "succeeded", + "borrowed": true, + "source": { + "uri": "boots/baai-bge-m3/l4-cold-with-bound-1.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000065" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "snapshot-cold", + "bootSeconds": 8.5, + "outcome": "succeeded", + "source": { + "uri": "boots/baai-bge-m3/l4-snapshot-cold-2.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000066" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "BAAI/bge-m3:sparse", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "bootSeconds": 88.1, + "outcome": "succeeded", + "source": { + "uri": "boots/baai-bge-m3-sparse/l4-cold-with-bound-3.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000067" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "BAAI/bge-m3:sparse", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "bootSeconds": 95.0, + "outcome": "succeeded", + "source": { + "uri": "boots/baai-bge-m3-sparse/l4-cold-with-bound-4.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000068" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-8B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "warmth": "cold-with-bound", + "bootSeconds": 187.4, + "outcome": "succeeded", + "source": { + "uri": "boots/qwen-qwen3-8b/a100-80gb-cold-with-bound-5.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000069" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-8B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "bootSeconds": 240.0, + "outcome": "succeeded", + "source": { + "uri": "boots/qwen-qwen3-8b/l4-cold-with-bound-6.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006a" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "bootSeconds": 240.0, + "outcome": "succeeded", + "source": { + "uri": "boots/qwen-qwen3-embedding-4b/l4-cold-with-bound-7.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006b" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "snapshot-cold", + "bootSeconds": 45.0, + "outcome": "succeeded", + "source": { + "uri": "boots/qwen-qwen3-embedding-4b/l4-snapshot-cold-8.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006c" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "warmth": "cold-with-bound", + "bootSeconds": 900.0, + "outcome": "timed-out", + "source": { + "uri": "boots/qwen-qwen3-embedding-4b/a100-80gb-cold-with-bound-9.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006d" + } + }, + { + "kind": "placement-boot-evidence", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-40GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "warmth": "cold-with-bound", + "bootSeconds": 70.0, + "outcome": "succeeded", + "source": { + "uri": "boots/qwen-qwen3-reranker-0.6b/a100-80gb-cold-with-bound-10.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006e" + } + } +] diff --git a/packages/sie_placement/tests/fixtures/planner/current-plan.json b/packages/sie_placement/tests/fixtures/planner/current-plan.json new file mode 100644 index 000000000..b51a64d28 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/current-plan.json @@ -0,0 +1,64 @@ +{ + "kind": "placement-plan", + "schemaVersion": 1, + "placement": "us-central1", + "lanes": [ + { + "name": "default-l4-default", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + { + "name": "qwen-qwen3-8b-a100-80gb", + "queuePool": "default", + "bundle": "sglang", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 32, + "maxContainers": 4, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + } + ], + "identities": [ + { + "identity": "BAAI/bge-m3", + "lane": "default-l4-default", + "warmth": "pinned" + }, + { + "identity": "Qwen/Qwen3-8B", + "lane": "qwen-qwen3-8b-a100-80gb", + "warmth": "cold-with-bound" + }, + { + "identity": "intfloat/e5-base-v2", + "lane": "default-l4-default", + "warmth": "pinned" + } + ] +} diff --git a/packages/sie_placement/tests/fixtures/planner/demand.json b/packages/sie_placement/tests/fixtures/planner/demand.json new file mode 100644 index 000000000..b00b366ee --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/demand.json @@ -0,0 +1,114 @@ +{ + "kind": "placement-demand", + "schemaVersion": 1, + "start": "2026-11-02T00:00:00Z", + "end": "2026-11-03T00:00:00Z", + "series": [ + { + "identity": "BAAI/bge-m3", + "operation": "encode", + "hours": [ + { + "start": "2026-11-02T09:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T10:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T11:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T12:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T13:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T14:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T15:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T16:00:00Z", + "requests": 20000 + }, + { + "start": "2026-11-02T17:00:00Z", + "requests": 20000 + } + ] + }, + { + "identity": "BAAI/bge-m3:sparse", + "operation": "encode", + "hours": [ + { + "start": "2026-11-02T11:00:00Z", + "requests": 3 + } + ] + }, + { + "identity": "Qwen/Qwen3-8B", + "operation": "generate", + "hours": [ + { + "start": "2026-11-02T14:00:00Z", + "requests": 12 + }, + { + "start": "2026-11-02T15:00:00Z", + "requests": 30 + } + ] + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "operation": "encode", + "hours": [ + { + "start": "2026-11-02T08:00:00Z", + "requests": 600 + }, + { + "start": "2026-11-02T09:00:00Z", + "requests": 600 + }, + { + "start": "2026-11-02T10:00:00Z", + "requests": 600 + }, + { + "start": "2026-11-02T11:00:00Z", + "requests": 600 + }, + { + "start": "2026-11-02T12:00:00Z", + "requests": 600 + }, + { + "start": "2026-11-02T13:00:00Z", + "requests": 600 + } + ] + }, + { + "identity": "intfloat/e5-base-v2", + "operation": "encode", + "hours": [ + { + "start": "2026-11-02T10:00:00Z", + "requests": 40 + } + ] + } + ] +} diff --git a/packages/sie_placement/tests/fixtures/planner/expected-plan.json b/packages/sie_placement/tests/fixtures/planner/expected-plan.json new file mode 100644 index 000000000..1edfd3444 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/expected-plan.json @@ -0,0 +1,134 @@ +{ + "kind": "placement-plan", + "schemaVersion": 1, + "placement": "us-central1", + "lanes": [ + { + "name": "baai-bge-m3-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + { + "name": "baai-bge-m3-sparse-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + }, + { + "name": "default-l4-default", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + { + "name": "qwen-qwen3-8b-a100-80gb", + "queuePool": "default", + "bundle": "sglang", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 32, + "maxContainers": 4, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + }, + { + "name": "qwen-qwen3-embedding-4b-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + } + ], + "identities": [ + { + "identity": "BAAI/bge-m3", + "lane": "baai-bge-m3-l4", + "warmth": "pinned" + }, + { + "identity": "BAAI/bge-m3:sparse", + "lane": "baai-bge-m3-sparse-l4", + "warmth": "cold-with-bound" + }, + { + "identity": "Qwen/Qwen3-8B", + "lane": "qwen-qwen3-8b-a100-80gb", + "warmth": "cold-with-bound" + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "lane": "qwen-qwen3-embedding-4b-l4", + "warmth": "pinned" + }, + { + "identity": "intfloat/e5-base-v2", + "lane": "default-l4-default", + "warmth": "pinned" + } + ] +} diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json new file mode 100644 index 000000000..ee6424bc9 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -0,0 +1,698 @@ +{ + "kind": "placement-report", + "schemaVersion": 1, + "placement": "us-central1", + "currency": "USD", + "parameters": { + "placement": "us-central1", + "maxContainers": 4, + "fallbackFloor": 2, + "cooldownS": 900, + "bufferContainers": 1, + "hoursPerMonth": 730.0, + "queuePool": "default", + "bundle": "default", + "bundles": { + "Qwen/Qwen3-8B": "sglang" + } + }, + "totals": { + "monthlyCost": 1947.88, + "envelopeMaxMonthlyCost": 10658.0 + }, + "identities": [ + { + "identity": "BAAI/bge-m3", + "status": "placed", + "serviceLevel": { + "comeAliveP95S": 0, + "warmP99Ms": 250, + "queueWaitP99Ms": 500 + }, + "demandRequests": 180000, + "lane": "baai-bge-m3-l4", + "chosen": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "warmth": "pinned", + "monthlyCost": 759.2, + "comeAliveS": 0.0, + "comeAliveBasis": "warm-floor", + "receipt": { + "uri": "receipts/baai-bge-m3/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000001" + } + }, + "peakContainers": 1, + "nextCheaper": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "warmth": "snapshot-cold", + "monthlyCost": 569.4, + "saving": 189.8, + "comeAliveS": 8.5, + "comeAliveBasis": "boot-evidence", + "blockedBy": [ + { + "field": "comeAliveP95S", + "bound": 0, + "measured": 8.5 + } + ], + "receipt": { + "uri": "receipts/baai-bge-m3/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000001" + } + }, + "current": { + "identity": "BAAI/bge-m3", + "lane": "default-l4-default", + "warmth": "pinned" + } + }, + { + "identity": "BAAI/bge-m3:sparse", + "status": "placed", + "serviceLevel": { + "comeAliveP95S": 120, + "warmP99Ms": 250, + "queueWaitP99Ms": 500 + }, + "demandRequests": 3, + "lane": "baai-bge-m3-sparse-l4", + "chosen": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "warmth": "cold-with-bound", + "monthlyCost": 47.45, + "comeAliveS": 95.0, + "comeAliveBasis": "boot-evidence", + "receipt": { + "uri": "receipts/baai-bge-m3-sparse/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000003" + } + }, + "peakContainers": 1 + }, + { + "identity": "Qwen/Qwen3-8B", + "status": "placed", + "serviceLevel": { + "comeAliveP95S": 600, + "ttftP99Ms": 4000, + "itlP99Ms": 80 + }, + "demandRequests": 42, + "lane": "qwen-qwen3-8b-a100-80gb", + "chosen": { + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 32, + "warmth": "cold-with-bound", + "monthlyCost": 382.03, + "comeAliveS": 187.4, + "comeAliveBasis": "boot-evidence", + "receipt": { + "uri": "receipts/qwen-qwen3-8b/a100-80gb-c32-65536.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000004" + } + }, + "peakContainers": 1, + "nextCheaper": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "warmth": "cold-with-bound", + "monthlyCost": 126.53, + "saving": 255.5, + "comeAliveS": 240.0, + "comeAliveBasis": "boot-evidence", + "blockedBy": [ + { + "field": "itlP99Ms", + "bound": 80, + "measured": 118 + } + ], + "receipt": { + "uri": "receipts/qwen-qwen3-8b/l4-c8-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000005" + } + }, + "current": { + "identity": "Qwen/Qwen3-8B", + "lane": "qwen-qwen3-8b-a100-80gb", + "warmth": "cold-with-bound" + } + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "status": "placed", + "serviceLevel": { + "comeAliveP95S": 30, + "warmP99Ms": 300 + }, + "demandRequests": 3600, + "lane": "qwen-qwen3-embedding-4b-l4", + "chosen": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "warmth": "pinned", + "monthlyCost": 759.2, + "comeAliveS": 0.0, + "comeAliveBasis": "warm-floor", + "receipt": { + "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000007" + } + }, + "peakContainers": 1, + "nextCheaper": { + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "warmth": "snapshot-cold", + "monthlyCost": 379.6, + "saving": 379.6, + "comeAliveS": 45.0, + "comeAliveBasis": "boot-evidence", + "blockedBy": [ + { + "field": "comeAliveP95S", + "bound": 30, + "measured": 45.0 + } + ], + "receipt": { + "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000007" + } + } + }, + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "status": "unplaced", + "serviceLevel": { + "comeAliveP95S": 90, + "warmP99Ms": 400 + }, + "demandRequests": 0, + "unplacedReason": "no-receipt" + }, + { + "identity": "intfloat/e5-base-v2", + "status": "kept", + "serviceLevel": { + "comeAliveP95S": 0, + "warmP99Ms": 250, + "queueWaitP99Ms": 500 + }, + "demandRequests": 40, + "lane": "default-l4-default", + "unplacedReason": "no-receipt", + "current": { + "identity": "intfloat/e5-base-v2", + "lane": "default-l4-default", + "warmth": "pinned" + } + } + ], + "lanes": [ + { + "name": "baai-bge-m3-l4", + "identities": [ + "BAAI/bge-m3" + ], + "hourlyCost": 1.04, + "floorMonthlyCost": 759.2, + "envelopeMaxMonthlyCost": 1518.4 + }, + { + "name": "baai-bge-m3-sparse-l4", + "identities": [ + "BAAI/bge-m3:sparse" + ], + "hourlyCost": 1.04, + "floorMonthlyCost": 0.0, + "envelopeMaxMonthlyCost": 1518.4 + }, + { + "name": "default-l4-default", + "identities": [ + "intfloat/e5-base-v2" + ], + "hourlyCost": 1.04, + "floorMonthlyCost": 759.2, + "envelopeMaxMonthlyCost": 1518.4 + }, + { + "name": "qwen-qwen3-8b-a100-80gb", + "identities": [ + "Qwen/Qwen3-8B" + ], + "hourlyCost": 3.14, + "floorMonthlyCost": 0.0, + "envelopeMaxMonthlyCost": 4584.4 + }, + { + "name": "qwen-qwen3-embedding-4b-l4", + "identities": [ + "Qwen/Qwen3-Embedding-4B" + ], + "hourlyCost": 1.04, + "floorMonthlyCost": 759.2, + "envelopeMaxMonthlyCost": 1518.4 + } + ], + "measurementRequests": [ + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 0.0 + }, + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "h100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 0.0 + }, + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "refused", + "costBound": 0.0 + }, + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "l40s", + "gpuProduct": "NVIDIA-L40S" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 0.0 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "measure": [ + "receipt" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 1825.0 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "h100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3" + }, + "measure": [ + "receipt" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 2920.0 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "measure": [ + "receipt" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 584.0 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "l40s", + "gpuProduct": "NVIDIA-L40S" + }, + "measure": [ + "receipt" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 876.0 + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l40s", + "gpuProduct": "NVIDIA-L40S" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 438.0, + "savingBound": 321.2 + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 32768 + }, + "measure": [ + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 467.2, + "savingBound": 292.0 + }, + { + "identity": "Qwen/Qwen3-8B", + "cell": { + "machineProfile": "l40s", + "gpuProduct": "NVIDIA-L40S" + }, + "measure": [ + "receipt", + "boot-evidence" + ], + "warmth": "cold-with-bound", + "reason": "unmeasured", + "costBound": 146.0, + "savingBound": 236.03 + } + ], + "comeAliveLevers": [ + { + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "comeAliveS": 240.0, + "comeAliveBasis": "boot-evidence", + "comeAliveBoundS": 30, + "monthlyCost": 379.6, + "savingUnderBound": 379.6, + "permittedBoundS": 240, + "savingAtPermittedBound": 379.6 + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "snapshot-cold", + "comeAliveS": 45.0, + "comeAliveBasis": "boot-evidence", + "comeAliveBoundS": 30, + "monthlyCost": 379.6, + "savingUnderBound": 379.6, + "permittedBoundS": 45, + "savingAtPermittedBound": 379.6 + }, + { + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "cold-with-bound", + "comeAliveS": 95.0, + "comeAliveBasis": "borrowed-boot-evidence", + "comeAliveBoundS": 0, + "monthlyCost": 569.4, + "savingUnderBound": 0.0, + "permittedBoundS": 95, + "savingAtPermittedBound": 189.8 + }, + { + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "warmth": "snapshot-cold", + "comeAliveS": 8.5, + "comeAliveBasis": "boot-evidence", + "comeAliveBoundS": 0, + "monthlyCost": 569.4, + "savingUnderBound": 0.0, + "permittedBoundS": 9, + "savingAtPermittedBound": 189.8 + } + ], + "queueWaitUnattained": [ + "BAAI/bge-m3", + "BAAI/bge-m3:sparse", + "intfloat/e5-base-v2" + ], + "refusals": [ + { + "record": "boot-evidence", + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-40GB" + }, + "source": { + "uri": "boots/qwen-qwen3-reranker-0.6b/a100-80gb-cold-with-bound-10.json", + "sha256": "000000000000000000000000000000000000000000000000000000000000006e" + }, + "reason": "realized GPU product 'NVIDIA-A100-SXM4-40GB' differs from 'NVIDIA-A100-SXM4-80GB', the product machine profile 'a100-80gb' requests" + }, + { + "record": "receipt", + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-A10G" + }, + "source": { + "uri": "receipts/qwen-qwen3-reranker-0.6b/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000006" + }, + "reason": "realized GPU product 'NVIDIA-A10G' differs from 'NVIDIA-L4', the product machine profile 'l4' requests" + } + ], + "changes": { + "identities": [ + { + "identity": "BAAI/bge-m3", + "before": { + "identity": "BAAI/bge-m3", + "lane": "default-l4-default", + "warmth": "pinned" + }, + "after": { + "identity": "BAAI/bge-m3", + "lane": "baai-bge-m3-l4", + "warmth": "pinned" + } + }, + { + "identity": "BAAI/bge-m3:sparse", + "after": { + "identity": "BAAI/bge-m3:sparse", + "lane": "baai-bge-m3-sparse-l4", + "warmth": "cold-with-bound" + } + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "after": { + "identity": "Qwen/Qwen3-Embedding-4B", + "lane": "qwen-qwen3-embedding-4b-l4", + "warmth": "pinned" + } + } + ], + "lanes": [ + { + "name": "baai-bge-m3-l4", + "after": { + "name": "baai-bge-m3-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + } + }, + { + "name": "baai-bge-m3-sparse-l4", + "after": { + "name": "baai-bge-m3-sparse-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + } + }, + { + "name": "qwen-qwen3-embedding-4b-l4", + "after": { + "name": "qwen-qwen3-embedding-4b-l4", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + } + } + ] + } +} diff --git a/packages/sie_placement/tests/fixtures/planner/price-list.json b/packages/sie_placement/tests/fixtures/planner/price-list.json new file mode 100644 index 000000000..08d835d8a --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/price-list.json @@ -0,0 +1,45 @@ +{ + "kind": "placement-price-list", + "schemaVersion": 1, + "currency": "USD", + "machineProfiles": [ + { + "name": "l4", + "gpuProduct": "NVIDIA-L4", + "acceleratorPerHour": 0.8, + "corePerHour": 0.04, + "memoryGiBPerHour": 0.005 + }, + { + "name": "l40s", + "gpuProduct": "NVIDIA-L40S", + "acceleratorPerHour": 1.2, + "corePerHour": 0.04, + "memoryGiBPerHour": 0.005 + }, + { + "name": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB", + "acceleratorPerHour": 2.5, + "corePerHour": 0.04, + "memoryGiBPerHour": 0.005 + }, + { + "name": "h100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3", + "acceleratorPerHour": 4.0, + "corePerHour": 0.04, + "memoryGiBPerHour": 0.005 + } + ], + "placements": [ + { + "name": "us-central1", + "regionMultiplier": 1 + }, + { + "name": "europe-west4", + "regionMultiplier": 1.25 + } + ] +} diff --git a/packages/sie_placement/tests/fixtures/planner/receipts.json b/packages/sie_placement/tests/fixtures/planner/receipts.json new file mode 100644 index 000000000..c29fe2621 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/receipts.json @@ -0,0 +1,295 @@ +[ + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 40 + }, + "latency": { + "p50Ms": 110.2, + "p95Ms": 171.9, + "p99Ms": 214 + }, + "source": { + "uri": "receipts/baai-bge-m3/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000001" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 32, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 120 + }, + "latency": { + "p50Ms": 61, + "p95Ms": 98, + "p99Ms": 120 + }, + "source": { + "uri": "receipts/baai-bge-m3/a100-80gb-c32-65536.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000002" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "BAAI/bge-m3:sparse", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 40 + }, + "latency": { + "p50Ms": 101, + "p99Ms": 190 + }, + "source": { + "uri": "receipts/baai-bge-m3-sparse/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000003" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-8B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 32, + "requestShape": { + "operation": "generate", + "itemsPerRequest": 1, + "inputTokensPerItem": 1024, + "outputTokens": 256 + }, + "throughput": { + "requestsPerSecond": 3.1, + "outputTokensPerSecond": 793.6 + }, + "ttft": { + "p50Ms": 310, + "p99Ms": 1480 + }, + "itl": { + "p50Ms": 24.5, + "p95Ms": 31, + "p99Ms": 44.8 + }, + "source": { + "uri": "receipts/qwen-qwen3-8b/a100-80gb-c32-65536.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000004" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-8B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "requestShape": { + "operation": "generate", + "itemsPerRequest": 1, + "inputTokensPerItem": 1024, + "outputTokens": 256 + }, + "throughput": { + "requestsPerSecond": 0.9, + "outputTokensPerSecond": 230.4 + }, + "ttft": { + "p50Ms": 820, + "p99Ms": 3900 + }, + "itl": { + "p50Ms": 61, + "p99Ms": 118 + }, + "source": { + "uri": "receipts/qwen-qwen3-8b/l4-c8-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000005" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Reranker-0.6B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-A10G" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "requestShape": { + "operation": "score", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 55 + }, + "latency": { + "p99Ms": 180 + }, + "source": { + "uri": "receipts/qwen-qwen3-reranker-0.6b/l4-c16-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000006" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 12 + }, + "latency": { + "p50Ms": 140, + "p99Ms": 280 + }, + "source": { + "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-16384.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000007" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 32768 + }, + "concurrency": 8, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 14 + }, + "latency": { + "p50Ms": 131, + "p99Ms": 262 + }, + "source": { + "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-32768.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000008" + } + }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "Qwen/Qwen3-Embedding-4B", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 16, + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 40 + }, + "latency": { + "p50Ms": 70, + "p99Ms": 150 + }, + "source": { + "uri": "receipts/qwen-qwen3-embedding-4b/a100-80gb-c16-65536.json", + "sha256": "0000000000000000000000000000000000000000000000000000000000000009" + } + } +] diff --git a/packages/sie_placement/tests/fixtures/planner/service-levels.json b/packages/sie_placement/tests/fixtures/planner/service-levels.json new file mode 100644 index 000000000..7a0da5891 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/planner/service-levels.json @@ -0,0 +1,48 @@ +{ + "kind": "placement-service-levels", + "schemaVersion": 1, + "tiers": [ + { + "name": "interactive-retrieval", + "comeAliveP95S": 0, + "warmP99Ms": 250, + "queueWaitP99Ms": 500 + }, + { + "name": "batch-generation", + "comeAliveP95S": 600, + "ttftP99Ms": 4000, + "itlP99Ms": 120 + } + ], + "identities": [ + { + "identity": "BAAI/bge-m3", + "tier": "interactive-retrieval" + }, + { + "identity": "BAAI/bge-m3:sparse", + "tier": "interactive-retrieval", + "comeAliveP95S": 120 + }, + { + "identity": "Qwen/Qwen3-8B", + "tier": "batch-generation", + "itlP99Ms": 80 + }, + { + "identity": "Qwen/Qwen3-Reranker-0.6B", + "comeAliveP95S": 90, + "warmP99Ms": 400 + }, + { + "identity": "intfloat/e5-base-v2", + "tier": "interactive-retrieval" + }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "comeAliveP95S": 30, + "warmP99Ms": 300 + } + ] +} diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py new file mode 100644 index 000000000..27e8a02df --- /dev/null +++ b/packages/sie_placement/tests/test_levers.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from sie_placement.planner import PlannerParameters, plan +from sie_placement.report import ComeAliveBasis +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + PriceList, + Receipt, + RequestShape, + ServiceLevel, + ServiceLevels, + SourceRef, + Throughput, + Warmth, +) + +START = datetime(2026, 11, 2, tzinfo=UTC) +MONTH = 730.0 +SCALE = MONTH / 24 +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +WIDE = ContainerShape(gpu_count=1, cpu_cores=8, memory_mib=32768) +PRICES = PriceList( + currency="USD", + machine_profiles=( + MachineProfilePrice( + name="l4", gpu_product="NVIDIA-L4", accelerator_per_hour=1.0, core_per_hour=0.1, memory_gib_per_hour=0.0 + ), + MachineProfilePrice( + name="a100-80gb", + gpu_product="NVIDIA-A100-SXM4-80GB", + accelerator_per_hour=3.0, + core_per_hour=0.1, + memory_gib_per_hour=0.0, + ), + ), + placements=(Placement(name="test", region_multiplier=1),), +) +PARAMETERS = PlannerParameters(placement="test", max_containers=4, fallback_floor=2, cooldown_s=900) +L4_HOURLY = 1.0 + 4 * 0.1 + + +def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8): + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=10.0), + latency=Percentiles(p99_ms=p99), + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}"), + ) + + +def boot(identity, seconds, warmth=Warmth.COLD_WITH_BOUND, cell=L4, *, shape=SHAPE, borrowed=False): + return BootEvidence( + identity=identity, + cell=cell, + container_shape=shape, + warmth=warmth, + boot_seconds=seconds, + outcome=BootOutcome.SUCCEEDED, + borrowed=borrowed, + source=SourceRef(uri=f"boots/{identity}/{cell.machine_profile}/{warmth.value}"), + ) + + +def busy(*identities, hours=6): + return Demand( + start=START, + end=START + timedelta(hours=24), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple(DemandHour(start=START + timedelta(hours=hour), requests=10) for hour in range(hours)), + ) + for identity in identities + ), + ) + + +def run(bounds, receipts, boots, demand=None): + service_levels = ServiceLevels( + identities=tuple( + IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + for identity, bound in bounds.items() + ) + ) + return plan(service_levels, receipts, boots, PRICES, demand or busy(*bounds), None, PARAMETERS).report + + +def cold_cost(hours=6): + return L4_HOURLY * hours * 2 * SCALE + + +def test_a_zero_bound_lever_saves_nothing_under_its_bound_and_is_priced_at_the_bound_it_permits(): + report = run({"org/warm": 0}, [receipt("org/warm")], [boot("org/warm", 95.2, borrowed=True)]) + (lever,) = report.come_alive_levers + assert (lever.identity, lever.cell, lever.container_shape, lever.warmth) == ( + "org/warm", + L4, + SHAPE, + Warmth.COLD_WITH_BOUND, + ) + assert (lever.bound_s, lever.come_alive_s, lever.permitted_bound_s) == (0, 95.2, 96) + assert lever.come_alive_basis is ComeAliveBasis.BORROWED_BOOT_EVIDENCE + assert lever.saving_under_bound == 0 + assert lever.monthly_cost == pytest.approx(cold_cost()) + assert lever.saving_at_permitted_bound == pytest.approx(L4_HOURLY * MONTH - cold_cost()) + + +def test_a_finite_bound_lever_saves_the_same_under_its_bound_and_at_the_bound_it_permits(): + report = run({"org/slow": 60}, [receipt("org/slow")], [boot("org/slow", 240.0)]) + (lever,) = report.come_alive_levers + assert (lever.bound_s, lever.permitted_bound_s) == (60, 240) + assert lever.saving_under_bound == pytest.approx(L4_HOURLY * MONTH - cold_cost()) + assert lever.saving_at_permitted_bound == lever.saving_under_bound + + +@pytest.mark.parametrize("seconds", [0.0, 0.4]) +def test_a_sub_second_restore_permits_a_one_second_bound(seconds): + report = run({"org/warm": 0}, [receipt("org/warm")], [boot("org/warm", seconds, Warmth.SNAPSHOT_COLD)]) + assert report.come_alive_levers[0].permitted_bound_s == 1 + + +def test_levers_rank_by_the_saving_under_the_bound_then_at_the_permitted_bound(): + bounds = {"org/always-warm": 0, "org/small-saving": 60, "org/big-saving": 60, "org/zero-quiet": 0} + receipts = [receipt(identity) for identity in bounds] + boots = [boot(identity, 120.0) for identity in bounds] + demand = Demand( + start=START, + end=START + timedelta(hours=24), + series=( + *busy("org/small-saving", hours=8).series, + *busy("org/big-saving", hours=2).series, + *busy("org/always-warm", hours=1).series, + *busy("org/zero-quiet", hours=4).series, + ), + ) + report = run(bounds, receipts, boots, demand) + ranked = [(lever.identity, lever.saving_under_bound > 0) for lever in report.come_alive_levers] + assert ranked == [ + ("org/big-saving", True), + ("org/small-saving", True), + ("org/always-warm", False), + ("org/zero-quiet", False), + ] + savings = [lever.saving_at_permitted_bound for lever in report.come_alive_levers] + assert savings[2] > savings[3] + + +def test_a_cell_that_misses_its_warm_bound_offers_no_lever(): + report = run( + {"org/model": 60}, [receipt("org/model", p99=400.0), receipt("org/model", A100)], [boot("org/model", 240.0)] + ) + assert report.come_alive_levers == () + assert report.identity("org/model").chosen.cell == A100 + + +def test_a_cold_posture_that_is_not_cheaper_offers_no_lever(): + report = run({"org/model": 60}, [receipt("org/model")], [boot("org/model", 240.0)], busy("org/model", hours=24)) + assert report.identity("org/model").chosen.warmth is Warmth.PINNED + assert report.come_alive_levers == () + + +def test_an_undeclared_bound_offers_no_lever(): + report = run({"org/model": None}, [receipt("org/model")], [boot("org/model", 240.0)]) + assert report.identity("org/model").chosen.warmth is Warmth.COLD_WITH_BOUND + assert report.come_alive_levers == () + + +def test_a_measured_come_alive_under_the_bound_is_a_choice_and_not_a_lever(): + report = run({"org/model": 60}, [receipt("org/model")], [boot("org/model", 30.0)]) + assert report.identity("org/model").chosen.warmth is Warmth.COLD_WITH_BOUND + assert report.come_alive_levers == () + + +def test_each_cell_shape_and_warmth_is_one_lever_at_its_cheapest_receipt(): + receipts = [ + receipt("org/model", concurrency=8), + receipt("org/model", concurrency=16), + receipt("org/model", shape=WIDE), + ] + boots = [ + boot("org/model", 240.0), + boot("org/model", 30.0 + 60.0, Warmth.SNAPSHOT_COLD), + boot("org/model", 300.0, shape=WIDE), + ] + report = run({"org/model": 60}, receipts, boots) + keys = [(lever.container_shape, lever.warmth) for lever in report.come_alive_levers] + assert sorted(keys, key=str) == sorted( + [(SHAPE, Warmth.COLD_WITH_BOUND), (SHAPE, Warmth.SNAPSHOT_COLD), (WIDE, Warmth.COLD_WITH_BOUND)], key=str + ) + assert len(keys) == len(set(keys)) + by_key = {(lever.container_shape, lever.warmth): lever for lever in report.come_alive_levers} + assert by_key[(SHAPE, Warmth.COLD_WITH_BOUND)].monthly_cost == pytest.approx(cold_cost()) + assert by_key[(WIDE, Warmth.COLD_WITH_BOUND)].monthly_cost == pytest.approx((1.0 + 8 * 0.1) * 6 * 2 * SCALE) + assert by_key[(SHAPE, Warmth.SNAPSHOT_COLD)].permitted_bound_s == 90 diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py new file mode 100644 index 000000000..90776c2bd --- /dev/null +++ b/packages/sie_placement/tests/test_planner.py @@ -0,0 +1,893 @@ +from __future__ import annotations + +import hashlib +import json +import math +import random +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from sie_placement.planner import ( + PlannerInputError, + PlannerParameters, + container_hourly_cost, + envelope_for, + plan, +) +from sie_placement.report import ( + Blocker, + ComeAliveBasis, + IdentityStatus, + Measurement, + RefusedRecord, + RequestReason, + UnplacedReason, + render, +) +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + RequestShape, + SchemaError, + ServiceLevel, + ServiceLevels, + SourceRef, + Throughput, + Warmth, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "planner" +START = datetime(2026, 11, 2, tzinfo=UTC) +WINDOW_HOURS = 24 +MONTH = 730.0 +SCALE = MONTH / WINDOW_HOURS + +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +L40S = Cell(machine_profile="l40s", gpu_product="NVIDIA-L40S") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +WIDE = ContainerShape(gpu_count=1, cpu_cores=8, memory_mib=32768) +IDENTITY = "org/model" + +ESTATE_PARAMETERS = PlannerParameters( + placement="us-central1", + max_containers=4, + fallback_floor=2, + cooldown_s=900, + buffer_containers=1, + bundles={"Qwen/Qwen3-8B": "sglang"}, +) + + +def profile(cell: Cell, accelerator: float, core: float = 0.0) -> MachineProfilePrice: + return MachineProfilePrice( + name=cell.machine_profile, + gpu_product=cell.gpu_product, + accelerator_per_hour=accelerator, + core_per_hour=core, + memory_gib_per_hour=0.0, + ) + + +def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=()) -> PriceList: + profiles = [profile(cell, rate, core) for cell, rate in ((L4, l4), (A100, a100)) if rate is not None] + return PriceList( + currency="USD", + machine_profiles=(*profiles, *extra), + placements=(Placement(name="test", region_multiplier=multiplier),), + ) + + +L4_ONLY = price_list(a100=None) + + +def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0, uri=None) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=rps), + latency=Percentiles(p99_ms=p99), + source=SourceRef(uri=uri or f"receipts/{identity}/{cell.machine_profile}-{concurrency}-{shape.memory_mib}"), + ) + + +def generation_receipt(identity=IDENTITY, cell=A100, *, ttft=1000.0, itl=40.0, rps=2.0) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=SHAPE, + concurrency=16, + request_shape=RequestShape(operation=Operation.GENERATE, items_per_request=1, output_tokens=256), + throughput=Throughput(requests_per_second=rps, output_tokens_per_second=rps * 256), + ttft=Percentiles(p99_ms=ttft), + itl=Percentiles(p99_ms=itl), + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}-generate"), + ) + + +def boot( + identity=IDENTITY, + cell=L4, + warmth=Warmth.COLD_WITH_BOUND, + seconds=60.0, + *, + outcome=BootOutcome.SUCCEEDED, + borrowed=False, + shape=SHAPE, + uri="boots/example.json", +) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=shape, + warmth=warmth, + boot_seconds=seconds, + outcome=outcome, + borrowed=borrowed, + source=SourceRef(uri=uri), + ) + + +def levels(*entries: tuple[str, ServiceLevel]) -> ServiceLevels: + return ServiceLevels(identities=tuple(IdentityServiceLevel(identity=name, level=lvl) for name, lvl in entries)) + + +def level(**fields) -> ServiceLevels: + return levels((IDENTITY, ServiceLevel(**fields))) + + +def demand(hourly=None) -> Demand: + return Demand( + start=START, + end=START + timedelta(hours=WINDOW_HOURS), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=requests) + for hour, requests in sorted(counts.items()) + ), + ) + for identity, counts in sorted((hourly or {}).items()) + ), + ) + + +def parameters(**overrides) -> PlannerParameters: + values = {"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides + return PlannerParameters(**values) + + +def run(service_levels, receipts=(), boots=(), *, prices=None, hourly=None, current=None, **overrides): + return plan( + service_levels, + receipts, + boots, + prices or price_list(), + demand(hourly), + current, + parameters(**overrides), + ) + + +def single_lane_plan(name="shared", warmth=Warmth.PINNED, cell=L4, identity=IDENTITY) -> Plan: + floor = 1 if warmth is Warmth.PINNED else 0 + return Plan( + placement="test", + lanes=( + PlanLane( + name=name, + queue_pool="default", + bundle="default", + cell=cell, + container_shape=SHAPE, + concurrency=8, + max_containers=2, + envelope=Envelope(floor_min=floor, floor_max=max(floor, 1)), + ), + ), + identities=(PlanIdentity(identity=identity, lane=name, warmth=warmth),), + ) + + +def load(name: str): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +def estate() -> dict: + return { + "service_levels": ServiceLevels.from_dict(load("service-levels.json")), + "receipts": [Receipt.from_dict(document) for document in load("receipts.json")], + "boot_evidence": [BootEvidence.from_dict(document) for document in load("boot-evidence.json")], + "price_list": PriceList.from_dict(load("price-list.json")), + "demand": Demand.from_dict(load("demand.json")), + "current_plan": Plan.from_dict(load("current-plan.json")), + "parameters": ESTATE_PARAMETERS, + } + + +def rendered(result) -> tuple[str, str]: + return render(result.plan.to_dict()), render(result.report.to_dict()) + + +def test_container_hourly_cost_prices_accelerators_cores_and_memory(): + price = MachineProfilePrice( + name="l4", gpu_product="NVIDIA-L4", accelerator_per_hour=2.0, core_per_hour=0.05, memory_gib_per_hour=0.01 + ) + shape = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=20480) + assert container_hourly_cost(price, shape, 1.5) == pytest.approx(1.5 * (2 * 2.0 + 8 * 0.05 + 20 * 0.01)) + + +def test_a_pinned_cell_costs_its_floor_plus_the_hours_demand_needs_above_it(): + result = run(level(come_alive_p95_s=0), [receipt(rps=1.0)], prices=L4_ONLY, hourly={IDENTITY: {3: 9000, 4: 100}}) + report = result.report.identity(IDENTITY) + assert report.status is IdentityStatus.PLACED + assert report.chosen.warmth is Warmth.PINNED + assert report.chosen.come_alive_basis is ComeAliveBasis.WARM_FLOOR + assert report.chosen.monthly_cost == pytest.approx(MONTH + (3 - 1) * SCALE) + assert report.peak_containers == 3 + assert report.demand_requests == 9100 + + +def test_needed_containers_stop_at_the_reviewed_ceiling(): + result = run( + level(come_alive_p95_s=0), [receipt(rps=1.0)], prices=L4_ONLY, hourly={IDENTITY: {0: 36000}}, max_containers=2 + ) + report = result.report.identity(IDENTITY) + assert report.chosen.monthly_cost == pytest.approx(MONTH + (2 - 1) * SCALE) + assert report.peak_containers == 10 + + +def test_a_cold_cell_costs_its_work_and_the_tail_each_burst_holds(): + result = run( + level(come_alive_p95_s=120), [receipt(rps=10.0)], [boot()], prices=L4_ONLY, hourly={IDENTITY: {0: 1, 5: 10}} + ) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.warmth is Warmth.COLD_WITH_BOUND + assert chosen.come_alive_s == 60.0 + one_burst = 1 / 10.0 + 1 * 2 * 900 + capped_hour = 2 * 3600 + assert chosen.monthly_cost == pytest.approx((one_burst + capped_hour) / 3600 * SCALE) + + +def test_an_active_lane_without_a_fallback_floor_holds_one_container(): + result = run( + level(come_alive_p95_s=120), + [receipt(rps=10.0)], + [boot()], + prices=L4_ONLY, + hourly={IDENTITY: {0: 2}}, + fallback_floor=0, + cooldown_s=60, + ) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.monthly_cost == pytest.approx((2 / 10.0 + 2 * 1 * 60) / 3600 * SCALE) + + +def test_the_active_floor_never_exceeds_the_ceiling(): + result = run( + level(come_alive_p95_s=120), + [receipt(rps=10.0)], + [boot()], + prices=L4_ONLY, + hourly={IDENTITY: {0: 1}}, + fallback_floor=3, + max_containers=1, + ) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.monthly_cost == pytest.approx((1 / 10.0 + 1 * 1 * 900) / 3600 * SCALE) + + +def test_an_idle_cold_cell_costs_nothing_and_its_lane_keeps_no_floor(): + result = run(level(come_alive_p95_s=120), [receipt()], [boot()], prices=L4_ONLY) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.warmth is Warmth.COLD_WITH_BOUND + assert chosen.monthly_cost == 0 + (lane,) = result.plan.lanes + assert lane.envelope == Envelope(floor_min=0, floor_max=0) + assert result.plan.identities == (PlanIdentity(identity=IDENTITY, lane=lane.name, warmth=Warmth.COLD_WITH_BOUND),) + + +def test_a_cold_posture_without_boot_evidence_is_a_measurement_request(): + result = run(level(come_alive_p95_s=120), [receipt()], prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + (request,) = result.report.measurement_requests + assert request.measure == (Measurement.BOOT_EVIDENCE,) + assert request.warmth is Warmth.COLD_WITH_BOUND + assert request.reason is RequestReason.UNMEASURED + assert (request.cell, request.container_shape) == (L4, SHAPE) + assert request.cost_bound == 0 + assert request.saving_bound == pytest.approx(MONTH) + + +def test_a_failed_boot_is_not_selectable_and_asks_for_a_new_measurement(): + boots = [boot(seconds=900.0, outcome=BootOutcome.TIMED_OUT)] + result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + (request,) = result.report.measurement_requests + assert request.reason is RequestReason.FAILED + assert result.report.come_alive_levers == () + + +def test_a_failed_boot_disqualifies_the_successful_ones_beside_it(): + boots = [boot(seconds=40.0, uri="boots/1.json"), boot(seconds=0.0, outcome=BootOutcome.FAILED, uri="boots/2.json")] + result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + + +def test_the_slowest_successful_boot_stands_for_the_percentile(): + boots = [boot(seconds=80.0, uri="boots/1.json"), boot(seconds=130.0, uri="boots/2.json")] + result = run(level(come_alive_p95_s=120), [receipt()], boots, prices=L4_ONLY) + report = result.report.identity(IDENTITY) + assert report.chosen.warmth is Warmth.PINNED + assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=120, measured=130.0),) + + +def test_own_boot_evidence_replaces_borrowed_evidence(): + borrowed = boot(seconds=150.0, borrowed=True, uri="boots/borrowed.json") + own = boot(seconds=60.0, uri="boots/own.json") + result = run(level(come_alive_p95_s=120), [receipt()], [borrowed, own], prices=L4_ONLY) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, 60.0) + assert chosen.come_alive_basis is ComeAliveBasis.BOOT_EVIDENCE + failed = boot(seconds=0.0, outcome=BootOutcome.FAILED, uri="boots/own-failed.json") + result = run(level(come_alive_p95_s=200), [receipt()], [boot(seconds=60.0, borrowed=True), failed], prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + + +def test_borrowed_boot_evidence_stands_in_and_is_labelled(): + result = run(level(come_alive_p95_s=120), [receipt()], [boot(seconds=60.0, borrowed=True)], prices=L4_ONLY) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, 60.0) + assert chosen.come_alive_basis is ComeAliveBasis.BORROWED_BOOT_EVIDENCE + assert chosen.to_dict()["comeAliveBasis"] == "borrowed-boot-evidence" + + +def test_a_zero_come_alive_bound_never_selects_a_cold_posture(): + boots = [boot(seconds=0.0), boot(warmth=Warmth.SNAPSHOT_COLD, seconds=0.0, uri="boots/snapshot.json")] + result = run(level(come_alive_p95_s=0), [receipt()], boots, prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + assert result.report.measurement_requests == () + + +def test_an_undeclared_come_alive_bound_admits_any_measured_boot(): + result = run(level(warm_p99_ms=250), [receipt()], [boot(seconds=5000.0)], prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + + +def test_a_snapshot_restore_under_the_bound_beats_a_slow_boot(): + boots = [boot(seconds=300.0), boot(warmth=Warmth.SNAPSHOT_COLD, seconds=20.0, uri="boots/snapshot.json")] + result = run(level(come_alive_p95_s=60), [receipt()], boots, prices=L4_ONLY) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.warmth is Warmth.SNAPSHOT_COLD + assert chosen.come_alive_s == 20.0 + assert result.plan.identities[0].warmth is Warmth.SNAPSHOT_COLD + + +def test_equal_costs_prefer_the_faster_come_alive(): + boots = [boot(seconds=50.0), boot(warmth=Warmth.SNAPSHOT_COLD, seconds=20.0, uri="boots/snapshot.json")] + result = run(level(come_alive_p95_s=60), [receipt()], boots, prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.SNAPSHOT_COLD + + +def test_a_cell_whose_receipt_misses_the_warm_bound_is_never_chosen(): + receipts = [receipt(cell=L4, p99=300.0), receipt(cell=A100, p99=120.0)] + result = run(level(come_alive_p95_s=0, warm_p99_ms=250), receipts) + report = result.report.identity(IDENTITY) + assert report.chosen.cell == A100 + assert report.chosen.monthly_cost == pytest.approx(3 * MONTH) + assert report.next_cheaper.cell == L4 + assert report.next_cheaper.blocked_by == (Blocker(field="warmP99Ms", bound=250, measured=300.0),) + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + assert result.plan.lanes[0].cell == A100 + + +def test_generation_cells_meet_both_the_first_token_and_the_per_token_bound(): + receipts = [generation_receipt(cell=L4, ttft=2000.0, itl=95.0), generation_receipt(cell=A100, ttft=1200.0)] + result = run(level(come_alive_p95_s=0, ttft_p99_ms=1500, itl_p99_ms=80), receipts) + report = result.report.identity(IDENTITY) + assert report.chosen.cell == A100 + assert report.next_cheaper.blocked_by == ( + Blocker(field="ttftP99Ms", bound=1500, measured=2000.0), + Blocker(field="itlP99Ms", bound=80, measured=95.0), + ) + + +def test_a_receipt_whose_realized_product_differs_from_its_profile_is_refused(): + faster = Cell(machine_profile="l4", gpu_product="NVIDIA-L40S") + result = run(level(come_alive_p95_s=0), [receipt(cell=faster), receipt(cell=A100)]) + report = result.report.identity(IDENTITY) + assert report.chosen.cell == A100 + (refusal,) = result.report.refusals + assert refusal.record is RefusedRecord.RECEIPT + assert refusal.cell == faster + assert refusal.reason == ( + "realized GPU product 'NVIDIA-L40S' differs from 'NVIDIA-L4', the product machine profile 'l4' requests" + ) + (request,) = result.report.measurement_requests + assert (request.cell, request.reason, request.measure) == (L4, RequestReason.REFUSED, (Measurement.RECEIPT,)) + assert request.saving_bound == pytest.approx(2 * MONTH) + assert all(lane.cell != faster for lane in result.plan.lanes) + + +def test_boot_evidence_whose_realized_product_differs_from_its_profile_is_refused(): + other = Cell(machine_profile="l4", gpu_product="NVIDIA-A10G") + result = run(level(come_alive_p95_s=120), [receipt()], [boot(cell=other)], prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + (refusal,) = result.report.refusals + assert refusal.record is RefusedRecord.BOOT_EVIDENCE + (request,) = result.report.measurement_requests + assert (request.measure, request.reason) == ((Measurement.BOOT_EVIDENCE,), RequestReason.UNMEASURED) + + +def test_a_receipt_on_an_unpriced_profile_is_refused(): + unpriced = Cell(machine_profile="t4", gpu_product="NVIDIA-T4") + result = run(level(come_alive_p95_s=0), [receipt(cell=unpriced)], prices=L4_ONLY) + report = result.report.identity(IDENTITY) + assert report.status is IdentityStatus.UNPLACED + assert report.unplaced_reason is UnplacedReason.NO_RECEIPT + assert result.report.refusals[0].reason == "machine profile 't4' has no price" + (request,) = result.report.measurement_requests + assert (request.cell, request.reason, request.saving_bound) == (L4, RequestReason.UNMEASURED, None) + + +@pytest.mark.parametrize( + ("service_level", "candidate", "reason"), + [ + ( + ServiceLevel(ttft_p99_ms=1000, itl_p99_ms=50), + receipt(), + "the receipt reports one latency and the service level declares ttftP99Ms and itlP99Ms", + ), + ( + ServiceLevel(warm_p99_ms=250), + generation_receipt(cell=L4), + "the receipt reports ttft and itl and the service level declares warmP99Ms", + ), + ], +) +def test_a_receipt_that_cannot_answer_the_service_level_is_refused(service_level, candidate, reason): + result = run(levels((IDENTITY, service_level)), [candidate], prices=L4_ONLY) + assert result.report.identity(IDENTITY).status is IdentityStatus.UNPLACED + assert [refusal.reason for refusal in result.report.refusals] == [reason] + + +def test_records_of_identities_without_a_service_level_are_ignored(): + stranger = Cell(machine_profile="l4", gpu_product="NVIDIA-A10G") + result = run( + level(come_alive_p95_s=0), + [receipt(), receipt(identity="org/other", cell=stranger)], + [boot(identity="org/other", cell=stranger)], + prices=L4_ONLY, + ) + assert result.report.refusals == () + assert [entry.identity for entry in result.plan.identities] == [IDENTITY] + assert [entry.identity for entry in result.report.identities] == [IDENTITY] + + +def test_an_unmeasured_cell_is_bounded_by_its_accelerator_price_alone(): + result = run(level(come_alive_p95_s=0), [receipt(cell=A100)], prices=price_list(core=0.5)) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.monthly_cost == pytest.approx((3.0 + 4 * 0.5) * MONTH) + (request,) = result.report.measurement_requests + assert (request.cell, request.measure, request.warmth) == (L4, (Measurement.RECEIPT,), Warmth.PINNED) + assert request.cost_bound == pytest.approx(1.0 * MONTH) + assert request.saving_bound == pytest.approx((5.0 - 1.0) * MONTH) + + +def test_an_unmeasured_cell_under_a_finite_bound_is_bounded_by_its_burst_tails(): + result = run(level(come_alive_p95_s=300), [receipt(cell=A100)], hourly={IDENTITY: {0: 1}}) + requests = {request.cell: request for request in result.report.measurement_requests} + request = requests[L4] + assert request.measure == (Measurement.RECEIPT, Measurement.BOOT_EVIDENCE) + assert request.warmth is Warmth.COLD_WITH_BOUND + assert request.cost_bound == pytest.approx(1.0 * (1 * 2 * 900) / 3600 * SCALE) + assert requests[A100].measure == (Measurement.BOOT_EVIDENCE,) + + +def test_a_cell_that_cannot_save_is_not_requested(): + result = run(level(come_alive_p95_s=0), [receipt(cell=L4)]) + assert result.report.measurement_requests == () + + +def test_measurement_requests_rank_unplaced_identities_first_then_by_saving(): + service_levels = levels( + ("org/big", ServiceLevel(come_alive_p95_s=0)), + ("org/small", ServiceLevel(come_alive_p95_s=120)), + ("org/unplaced", ServiceLevel(come_alive_p95_s=0)), + ) + receipts = [receipt(identity="org/big", cell=A100), receipt(identity="org/small", cell=L4)] + result = run(service_levels, receipts) + ranked = [ + (request.identity, request.cell.machine_profile, request.measure, request.saving_bound) + for request in result.report.measurement_requests + ] + assert ranked == [ + ("org/unplaced", "a100-80gb", (Measurement.RECEIPT,), None), + ("org/unplaced", "l4", (Measurement.RECEIPT,), None), + ("org/big", "l4", (Measurement.RECEIPT,), pytest.approx(2 * MONTH)), + ("org/small", "a100-80gb", (Measurement.RECEIPT, Measurement.BOOT_EVIDENCE), pytest.approx(MONTH)), + ("org/small", "l4", (Measurement.BOOT_EVIDENCE,), pytest.approx(MONTH)), + ] + + +def test_the_next_cheaper_option_is_the_nearest_cheaper_one_and_names_its_bound(): + prices = price_list(extra=(profile(L40S, 2.0),)) + receipts = [receipt(cell=L4, p99=400.0), receipt(cell=L40S, p99=300.0), receipt(cell=A100, p99=100.0)] + result = run(level(come_alive_p95_s=0, warm_p99_ms=250), receipts, prices=prices) + report = result.report.identity(IDENTITY) + assert report.chosen.cell == A100 + assert report.next_cheaper.cell == L40S + assert report.next_cheaper.saving == pytest.approx(MONTH) + assert report.next_cheaper.blocked_by == (Blocker(field="warmP99Ms", bound=250, measured=300.0),) + + +def test_a_placement_with_nothing_cheaper_names_no_next_option(): + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY) + assert result.report.identity(IDENTITY).next_cheaper is None + + +@pytest.mark.parametrize( + ("warmth", "buffer", "fallback", "ceiling", "expected"), + [ + (Warmth.PINNED, 0, 2, 4, (1, 1)), + (Warmth.PINNED, 2, 2, 4, (1, 3)), + (Warmth.PINNED, 5, 2, 4, (1, 4)), + (Warmth.COLD_WITH_BOUND, 0, 2, 4, (0, 0)), + (Warmth.COLD_WITH_BOUND, 1, 2, 4, (0, 2)), + (Warmth.COLD_WITH_BOUND, 3, 2, 4, (0, 3)), + (Warmth.COLD_WITH_BOUND, 1, 0, 4, (0, 1)), + (Warmth.COLD_WITH_BOUND, 1, 3, 2, (0, 2)), + (Warmth.SNAPSHOT_COLD, 1, 2, 4, (0, 2)), + ], +) +def test_envelopes_add_the_declared_buffer_at_realizable_floors(warmth, buffer, fallback, ceiling, expected): + envelope = envelope_for( + warmth, parameters(buffer_containers=buffer, fallback_floor=fallback, max_containers=ceiling) + ) + assert (envelope.floor_min, envelope.floor_max) == expected + assert (envelope.buffer_min, envelope.scaledown_window_min_s) == (None, None) + + +def test_the_plan_carries_the_envelope_and_the_report_prices_its_maximum(): + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, buffer_containers=2) + (lane,) = result.plan.lanes + assert lane.envelope == Envelope(floor_min=1, floor_max=3) + assert (lane.max_containers, lane.concurrency, lane.container_shape) == (4, 8, SHAPE) + (lane_report,) = result.report.lanes + assert lane_report.hourly_cost == pytest.approx(1.0) + assert lane_report.floor_monthly_cost == pytest.approx(MONTH) + assert lane_report.envelope_max_monthly_cost == pytest.approx(3 * MONTH) + assert result.report.envelope_max_monthly_cost == pytest.approx(3 * MONTH) + assert result.report.monthly_cost == pytest.approx(MONTH) + + +def test_every_declared_queue_wait_bound_is_reported_unattained(): + service_levels = levels( + ("org/a", ServiceLevel(come_alive_p95_s=0, queue_wait_p99_ms=500)), + ("org/b", ServiceLevel(come_alive_p95_s=0)), + ("org/c", ServiceLevel(queue_wait_p99_ms=30000)), + ) + result = run(service_levels, [receipt(identity="org/a")], prices=L4_ONLY) + assert result.report.queue_wait_unattained == ("org/a", "org/c") + + +def test_an_identity_without_a_feasible_cell_keeps_its_current_lane(): + current = single_lane_plan() + result = run(level(come_alive_p95_s=0, warm_p99_ms=50), [receipt(p99=100.0)], prices=L4_ONLY, current=current) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert report.status is IdentityStatus.KEPT + assert report.unplaced_reason is UnplacedReason.WARM_BOUND_MISSED + assert report.lane == "shared" + assert report.current == current.identities[0] + assert result.report.unchanged + assert result.report.lanes[0].floor_monthly_cost == pytest.approx(MONTH) + + +def test_an_identity_without_a_feasible_cell_or_a_current_lane_is_left_out(): + result = run(level(come_alive_p95_s=0), [], prices=L4_ONLY) + assert (result.plan.lanes, result.plan.identities) == ((), ()) + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.UNPLACED, UnplacedReason.NO_RECEIPT) + assert result.report.monthly_cost == 0 + + +def test_lane_names_derive_from_the_identity_and_its_machine_profile(): + result = run(levels(("BAAI/bge-m3:sparse", ServiceLevel())), [receipt(identity="BAAI/bge-m3:sparse")]) + assert result.plan.lanes[0].name == "baai-bge-m3-sparse-l4" + + +def test_colliding_lane_names_take_an_identity_digest(): + service_levels = levels(("org/model", ServiceLevel()), ("org-model", ServiceLevel())) + receipts = [receipt(identity="org/model"), receipt(identity="org-model")] + result = run(service_levels, receipts, prices=L4_ONLY) + names = {entry.identity: entry.lane for entry in result.plan.identities} + assert names == { + identity: f"org-model-l4-{hashlib.sha256(identity.encode()).hexdigest()[:8]}" for identity in names + } + assert len(set(names.values())) == 2 + + +def test_a_derived_lane_name_never_takes_a_kept_lane_name(): + current = single_lane_plan(name="org-model-l4", identity="org/kept") + service_levels = levels(("org/kept", ServiceLevel(come_alive_p95_s=0)), (IDENTITY, ServiceLevel())) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current) + names = {entry.identity: entry.lane for entry in result.plan.identities} + assert names["org/kept"] == "org-model-l4" + assert names[IDENTITY] == f"org-model-l4-{hashlib.sha256(IDENTITY.encode()).hexdigest()[:8]}" + + +def test_bundles_and_the_queue_pool_come_from_the_parameters(): + service_levels = levels(("org/embed", ServiceLevel()), ("org/generate", ServiceLevel())) + result = run( + service_levels, + [receipt(identity="org/embed"), receipt(identity="org/generate")], + prices=L4_ONLY, + queue_pool="batch", + bundle="default", + bundles={"org/generate": "sglang"}, + ) + lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} + assert lanes == {"org-embed-l4": ("batch", "default"), "org-generate-l4": ("batch", "sglang")} + + +def test_equal_costs_prefer_the_current_placement(): + twin = Cell(machine_profile="l4-b", gpu_product="NVIDIA-L4") + prices = price_list(a100=None, extra=(profile(twin, 1.0),)) + receipts = [receipt(cell=L4), receipt(cell=twin)] + first = run(level(come_alive_p95_s=0), receipts, prices=prices) + assert first.plan.lanes[0].cell == L4 + current = single_lane_plan(name="org-model-l4-b", cell=twin) + second = run(level(come_alive_p95_s=0), receipts, prices=prices, current=current) + assert second.plan.lanes[0].cell == twin + assert second.report.identity(IDENTITY).current == current.identities[0] + + +def test_the_region_multiplier_scales_every_price(): + result = run(level(come_alive_p95_s=0), [receipt()], prices=price_list(a100=None, multiplier=1.25)) + assert result.report.identity(IDENTITY).chosen.monthly_cost == pytest.approx(1.25 * MONTH) + assert result.report.lanes[0].hourly_cost == pytest.approx(1.25) + + +@pytest.mark.parametrize( + ("overrides", "path"), + [ + ({"placement": ""}, "placement"), + ({"max_containers": 0}, "maxContainers"), + ({"fallback_floor": -1}, "fallbackFloor"), + ({"cooldown_s": -1}, "cooldownS"), + ({"buffer_containers": -1}, "bufferContainers"), + ({"hours_per_month": 0}, "hoursPerMonth"), + ({"hours_per_month": math.inf}, "hoursPerMonth"), + ({"queue_pool": "two words"}, "queuePool"), + ({"bundle": ""}, "bundle"), + ({"bundles": {"": "sglang"}}, "bundles"), + ({"bundles": {IDENTITY: ""}}, "bundles['org/model']"), + ], +) +def test_parameters_are_validated(overrides, path): + with pytest.raises(SchemaError) as caught: + parameters(**overrides) + assert caught.value.path == path + + +def test_parameters_echo_into_the_report(): + echoed = parameters(bundles={"org/b": "sglang", "org/a": "default"}).to_dict() + assert echoed == { + "placement": "test", + "maxContainers": 4, + "fallbackFloor": 2, + "cooldownS": 900, + "bufferContainers": 0, + "hoursPerMonth": 730.0, + "queuePool": "default", + "bundle": "default", + "bundles": {"org/a": "default", "org/b": "sglang"}, + } + assert list(echoed["bundles"]) == ["org/a", "org/b"] + + +def test_the_placement_must_be_in_the_price_list(): + with pytest.raises(PlannerInputError, match="placement 'elsewhere' is not in the price list"): + run(level(), [receipt()], placement="elsewhere") + + +def test_the_current_plan_must_be_for_the_same_placement(): + current = Plan(placement="other", lanes=(), identities=()) + with pytest.raises(PlannerInputError, match="the current plan is for placement 'other'"): + run(level(), [receipt()], current=current) + + +def test_the_estate_plan_and_report_are_byte_stable(): + plan_text, report_text = rendered(plan(**estate())) + assert plan_text == (FIXTURES / "expected-plan.json").read_text(encoding="utf-8") + assert report_text == (FIXTURES / "expected-report.json").read_text(encoding="utf-8") + assert rendered(plan(**estate())) == (plan_text, report_text) + + +def test_input_order_does_not_change_the_output(): + baseline = rendered(plan(**estate())) + shuffler = random.Random(11) # noqa: S311 -- reproducible test data + for _ in range(5): + inputs = estate() + shuffler.shuffle(inputs["receipts"]) + shuffler.shuffle(inputs["boot_evidence"]) + service_levels = inputs["service_levels"] + inputs["service_levels"] = ServiceLevels( + identities=tuple(reversed(service_levels.identities)), tiers=tuple(reversed(service_levels.tiers)) + ) + source = inputs["demand"] + inputs["demand"] = Demand(start=source.start, end=source.end, series=tuple(reversed(source.series))) + assert rendered(plan(**inputs)) == baseline + + +def test_replanning_the_plan_it_wrote_changes_nothing(): + first = plan(**estate()) + second = plan(**{**estate(), "current_plan": first.plan}) + assert second.plan == first.plan + assert render(second.plan.to_dict()) == render(first.plan.to_dict()) + assert second.report.unchanged + assert second.report.to_dict()["changes"] == {"identities": [], "lanes": []} + + +def test_the_estate_plan_parses_as_a_plan_document(): + result = plan(**estate()) + assert Plan.from_dict(json.loads(render(result.plan.to_dict()))) == result.plan + + +def test_the_estate_places_every_identity_on_a_measured_feasible_cell(): + inputs = estate() + result = plan(**inputs) + receipts = {document.source.uri: document for document in inputs["receipts"]} + for entry in result.report.identities: + if entry.status is not IdentityStatus.PLACED: + continue + chosen = entry.chosen + assert chosen.blocked_by == () + measured = receipts[chosen.receipt.uri] + assert measured.cell == chosen.cell + if measured.latency is not None and entry.service_level.warm_p99_ms is not None: + assert measured.latency.p99_ms <= entry.service_level.warm_p99_ms + if measured.itl is not None: + assert measured.ttft.p99_ms <= entry.service_level.ttft_p99_ms + assert measured.itl.p99_ms <= entry.service_level.itl_p99_ms + if chosen.warmth is not Warmth.PINNED: + assert entry.service_level.come_alive_p95_s > 0 + assert chosen.come_alive_s <= entry.service_level.come_alive_p95_s + + +def test_the_estate_report_states_its_findings(): + report = plan(**estate()).report + dense = report.identity("BAAI/bge-m3") + assert (dense.chosen.cell, dense.chosen.warmth) == (L4, Warmth.PINNED) + assert dense.chosen.monthly_cost == pytest.approx(1.04 * MONTH) + assert dense.next_cheaper.warmth is Warmth.SNAPSHOT_COLD + assert dense.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=0, measured=8.5),) + sparse = report.identity("BAAI/bge-m3:sparse") + assert (sparse.chosen.warmth, sparse.chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, 95.0) + assert sparse.chosen.monthly_cost == pytest.approx(1.04 * (3 / 40 + 3 * 2 * 900) / 3600 * SCALE) + generation = report.identity("Qwen/Qwen3-8B") + assert (generation.chosen.cell, generation.chosen.warmth) == (A100, Warmth.COLD_WITH_BOUND) + assert generation.chosen.monthly_cost == pytest.approx(3.14 * 2 * 2 * SCALE) + assert generation.next_cheaper.blocked_by == (Blocker(field="itlP99Ms", bound=80, measured=118.0),) + assert report.identity("Qwen/Qwen3-Reranker-0.6B").status is IdentityStatus.UNPLACED + assert report.identity("intfloat/e5-base-v2").status is IdentityStatus.KEPT + assert [refusal.record for refusal in report.refusals] == [RefusedRecord.BOOT_EVIDENCE, RefusedRecord.RECEIPT] + assert report.queue_wait_unattained == ("BAAI/bge-m3", "BAAI/bge-m3:sparse", "intfloat/e5-base-v2") + assert [change.identity for change in report.identity_changes] == [ + "BAAI/bge-m3", + "BAAI/bge-m3:sparse", + "Qwen/Qwen3-Embedding-4B", + ] + assert report.monthly_cost == pytest.approx( + sum(entry.chosen.monthly_cost for entry in report.identities if entry.chosen is not None) + ) + + +@pytest.mark.parametrize("seed", range(25)) +def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = [f"org/model-{index}" for index in range(rng.randint(1, 5))] + service_levels = levels( + *( + ( + identity, + ServiceLevel( + come_alive_p95_s=rng.choice([None, 0, 30, 120, 600]), + warm_p99_ms=rng.choice([None, 150, 250, 400]), + ), + ) + for identity in identities + ) + ) + receipts = [ + receipt( + identity=identity, + cell=cell, + concurrency=rng.choice([4, 8]), + rps=rng.choice([0.5, 5.0, 50.0]), + p99=rng.choice([100.0, 200.0, 300.0, 500.0]), + uri=f"receipts/{identity}/{cell.machine_profile}/{index}", + ) + for identity in identities + for index, cell in enumerate([L4, A100, L4]) + if rng.random() < 0.7 + ] + boots = [ + boot( + identity=identity, + cell=cell, + warmth=warmth, + seconds=rng.choice([5.0, 60.0, 200.0]), + outcome=rng.choice([BootOutcome.SUCCEEDED, BootOutcome.SUCCEEDED, BootOutcome.FAILED]), + borrowed=rng.random() < 0.3, + uri=f"boots/{identity}/{cell.machine_profile}/{warmth.value}/{index}", + ) + for identity in identities + for cell in (L4, A100) + for warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) + for index in range(2) + if rng.random() < 0.4 + ] + hourly = { + identity: {hour: rng.choice([0, 1, 50, 20000]) for hour in rng.sample(range(WINDOW_HOURS), rng.randint(0, 6))} + for identity in identities + } + prices = price_list(l4=rng.choice([0.5, 1.0]), a100=rng.choice([2.0, 3.0]), core=0.1) + overrides = { + "buffer_containers": rng.randint(0, 2), + "fallback_floor": rng.randint(0, 3), + "cooldown_s": rng.choice([0, 60, 900]), + } + result = run(service_levels, receipts, boots, prices=prices, hourly=hourly, **overrides) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + by_uri = {document.source.uri: document for document in receipts} + for entry in result.report.identities: + if entry.chosen is None: + continue + chosen = entry.chosen + assert chosen.blocked_by == () + bound = entry.service_level.warm_p99_ms + assert bound is None or by_uri[chosen.receipt.uri].latency.p99_ms <= bound + come_alive = entry.service_level.come_alive_p95_s + if chosen.warmth is not Warmth.PINNED: + assert come_alive is None or come_alive > 0 + assert come_alive is None or chosen.come_alive_s <= come_alive + if entry.next_cheaper is not None: + assert entry.next_cheaper.blocked_by + assert entry.next_cheaper.monthly_cost < chosen.monthly_cost + costs = {entry.identity: entry.chosen.monthly_cost for entry in result.report.identities if entry.chosen} + for lever in result.report.come_alive_levers: + assert lever.monthly_cost < costs[lever.identity] + assert lever.saving_at_permitted_bound >= lever.saving_under_bound >= 0 + for request in result.report.measurement_requests: + assert request.saving_bound is None or request.saving_bound > 0 + again = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) + assert again.plan == result.plan + assert again.report.unchanged + shuffled = list(receipts) + rng.shuffle(shuffled) + reordered = run(service_levels, shuffled, list(reversed(boots)), prices=prices, hourly=hourly, **overrides) + assert rendered(reordered) == rendered(result) diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index 2ca86915c..c05a873c3 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -334,6 +334,26 @@ def test_boot_evidence_keeps_failed_outcomes(): assert str(schema_error(BootEvidence.from_dict, data)) == "$.bootSeconds: must be at least 0, got -0.5" +def test_boot_evidence_marks_evidence_borrowed_from_another_environment(): + data = load("boot-evidence.json") + assert BootEvidence.from_dict(data).borrowed is False + data["borrowed"] = True + evidence = BootEvidence.from_dict(data) + assert evidence.borrowed is True + assert evidence.to_dict() == data + data["borrowed"] = False + evidence = BootEvidence.from_dict(data) + assert evidence.borrowed is False + assert "borrowed" not in evidence.to_dict() + + +@pytest.mark.parametrize(("value", "described"), [(1, "number 1"), ("true", "string 'true'")]) +def test_boot_evidence_borrowed_is_a_strict_boolean(value, described): + data = load("boot-evidence.json") + data["borrowed"] = value + assert str(schema_error(BootEvidence.from_dict, data)) == f"$.borrowed: expected a boolean, got {described}" + + def test_price_lists_validate_currency_rates_and_names(): data = load("price-list.json") data["currency"] = "usd" From 46d5f3340e7c177adac6bb9b225f627567c8b04f Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 01:41:26 +0100 Subject: [PATCH 02/40] fix(placement): keep current lanes and price every lane of the plan An identity whose choice is its current placement now stays on its current lane under its current name, beside whoever else that lane hosts. The current placement also has to match the queue pool and the bundle the parameters give the identity. Before this, the planner renamed an unchanged lane and split a shared current lane into new pinned floors, so an unchanged input could still produce a diff. Only lanes for new or moved placements get minted names, and a minted name widens its digest when a kept lane already holds it. Kept lanes stay within the reviewed ceiling. Costs are now per lane: each lane pays its floor once, plus its residents' demand above the floor. A kept identity's demand is priced at its slowest receipt for its lane's cell, shape and concurrency, under that lane's ceiling. The totals cover every lane. totals.unpricedLanes lists each lane whose cell has no price (with an unpricedReason) or that has a resident whose demand no receipt can price. Each identity reports its monthly cost on its lane. A kept identity's blockedBy names the bounds its kept placement is known to miss. The report also lists sharedChartLanes: every chart lane, by queue pool, machine profile and bundle, that more than one plan lane names. Boot seconds normalize to float, so an int and a float record of the same boot render the same bytes in either order. New tests cover demand summed over operations, saturated cold hours, the cheapest receipt for levers and boot requests under shuffled input, the concurrency match of the current placement, and snapshot restores, which are never requested. --- packages/sie_placement/README.md | 63 +++- .../src/sie_placement/planner.py | 245 ++++++++---- .../sie_placement/src/sie_placement/report.py | 57 ++- .../tests/fixtures/planner/expected-plan.json | 22 +- .../fixtures/planner/expected-report.json | 78 ++-- packages/sie_placement/tests/test_levers.py | 24 +- packages/sie_placement/tests/test_planner.py | 353 +++++++++++++++++- 7 files changed, 661 insertions(+), 181 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 643684cc7..cfeb962ed 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -89,7 +89,9 @@ Helm chart: `queuePool`, the `machineProfile` of its `cell`, and `bundle`. override. It is neither the `workers.pools` map key, which names a worker group, nor an SDK logical pool. `bundle` is a key of that worker group's `bundles` map. `name` is the lane's handle inside the plan, and each identity -refers to its lane by that name. +refers to its lane by that name. Two plan lanes may carry the same three +tokens, and so name one chart lane, which cannot give them separate envelopes +or concurrency; the planner's report lists every such chart lane. ## Envelope @@ -119,8 +121,9 @@ print(render(result.plan.to_dict()), render(result.report.to_dict())) `plan` is a pure function of its documents. The same inputs render to the same bytes in whatever order the receipts, boot evidence, identities and demand -series arrive, and a run whose `current_plan` is its own previous plan reports -no change. +series arrive. An identity whose choice is its current placement keeps its +current lane and that lane's name, so a run whose `current_plan` already holds +every choice, such as its own previous plan, reports no change. | Parameter | Meaning | | --- | --- | @@ -134,8 +137,8 @@ no change. ### Selection -The planner places each identity of the service-level document on its own -lane: +The planner chooses a cell and warmth for each identity of the service-level +document: 1. It refuses a receipt or boot record whose machine profile has no price, or whose realized GPU product differs from the product the price list gives @@ -148,12 +151,21 @@ lane: successful boot stands for the come-alive percentile, a boot that did not succeed withdraws the warmth, and the environment's own evidence replaces borrowed evidence. A `comeAliveP95S` of `0` admits `pinned` only. -4. The cheapest unblocked option wins. Ties go to the current plan's - placement, then to the faster come-alive. - -The planner does not select `lazy-resident`, `bridged` or `scheduled-warm`, -and does not put two identities on one lane. An identity with no unblocked -option keeps its current lane, or stays out of the plan when it has none. +4. The cheapest unblocked option wins. Ties go to the current placement, + then to the faster come-alive. + +The current placement is the identity's current lane when that lane has the +option's cell, container shape and concurrency, the identity's warmth is the +option's, and the lane's `queuePool` and `bundle` are the ones the parameters +give the identity. An identity whose choice is its current placement stays on +that lane under its name, beside whoever else the lane hosts, and the lane +takes the ceiling and envelope of the choice. Every other placed identity gets +a new lane of its own, named from the identity and its machine profile, with a +digest of the identity added when that name is taken. The planner does not +select `lazy-resident`, `bridged` or `scheduled-warm`, and does not move an +identity onto another identity's lane. An identity with no unblocked option +keeps its current lane, capped by the reviewed ceiling, or stays out of the +plan when it has none. ### Pricing @@ -170,6 +182,13 @@ demand window to `hours_per_month`. every request: the active floor for `cooldown_s`, at most the whole hour. The active floor is `fallback_floor` capped by the ceiling, and at least one. +A lane costs its floor all month once, whoever it hosts, plus what each +resident's demand needs beyond the floor: on a lane with a floor, the +containers an hour needs above it, and on a zero-floor lane, the work and the +tails a cold warmth pays. A placed identity's demand is priced at its chosen +receipt, and a kept identity's at the slowest receipt it has for its lane's +cell, container shape and concurrency, under that lane's own ceiling. + ### Envelope `floorMin` is 1 for `pinned` and 0 otherwise. No source measures queue wait @@ -183,14 +202,26 @@ only. The report renders as a `placement-report` document: +- `totals`: the monthly cost of every lane of the plan, the monthly cost of + their `floorMax`, and `unpricedLanes`, the lanes whose cost the totals leave + out in whole or in part. - `identities`: each identity's status (`placed`, `kept` or `unplaced`), - effective service level and demand, the chosen cell, warmth, monthly cost and - come-alive with its basis (`warm-floor`, `boot-evidence` or + effective service level and demand, its lane and `monthlyCost`, what its + demand costs there with the lane's floor, the chosen cell, warmth, monthly + cost and come-alive with its basis (`warm-floor`, `boot-evidence` or `borrowed-boot-evidence`), the peak containers its demand needs, and `nextCheaper`: the nearest cheaper option a service level blocks, its saving - and the bounds that block it. -- `lanes`: each lane's hourly cost, its floor's monthly cost and the monthly - cost of its `floorMax`, which a floor budget prices. + and the bounds that block it. A kept identity's `blockedBy` names the bounds + its kept placement is known to miss: its receipt's warm latencies, a measured + come-alive over the bound, or a cold warmth under a bound of `0`, which needs + no measurement. +- `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost + and the monthly cost of its `floorMax`, which a floor budget prices. A lane + on a cell the price list cannot price carries `unpricedReason` and no cost, + and `unpricedIdentities` lists residents whose demand has no receipt to price + it on the lane. +- `sharedChartLanes`: every chart lane, by `queuePool`, `machineProfile` and + `bundle`, that more than one plan lane names. - `measurementRequests`: options that lack evidence, ranked by the saving they could unlock. A profile without a receipt is bounded by price alone: one accelerator at its rate, kept warm or, where the come-alive bound allows, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 29b23f997..df445c623 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -27,6 +27,7 @@ RefusedRecord, Report, RequestReason, + SharedChartLane, UnplacedReason, ) from sie_placement.schema import ( @@ -160,11 +161,12 @@ def scale(self) -> float: def needed(requests: int, requests_per_second: float) -> int: return math.ceil(requests / (requests_per_second * SECONDS_PER_HOUR)) + def above_floor_hours(self, requests: Sequence[int], requests_per_second: float, floor: int) -> float: + above = sum(max(0, min(self.ceiling, self.needed(count, requests_per_second)) - floor) for count in requests) + return above * self.scale + def warm_container_hours(self, requests: Sequence[int], requests_per_second: float) -> float: - above = sum( - max(0, min(self.ceiling, self.needed(count, requests_per_second)) - PINNED_FLOOR) for count in requests - ) - return PINNED_FLOOR * self.hours_per_month + above * self.scale + return PINNED_FLOOR * self.hours_per_month + self.above_floor_hours(requests, requests_per_second, PINNED_FLOOR) def cold_container_hours(self, requests: Sequence[int], requests_per_second: float | None) -> float: seconds = 0.0 @@ -244,8 +246,8 @@ def _slug(value: str) -> str: return _SLUG.sub("-", value.lower()).strip("-") -def _digest(value: str) -> str: - return hashlib.sha256(value.encode()).hexdigest()[:8] +def _digest(value: str, width: int = 8) -> str: + return hashlib.sha256(value.encode()).hexdigest()[:width] def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> str | None: @@ -302,7 +304,7 @@ def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: if any(record.outcome is not BootOutcome.SUCCEEDED for record in used): index[key] = _Boot(seconds=None, borrowed=not own) else: - index[key] = _Boot(seconds=max(record.boot_seconds for record in used), borrowed=not own) + index[key] = _Boot(seconds=float(max(record.boot_seconds for record in used)), borrowed=not own) return index @@ -316,13 +318,9 @@ def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: } -def _matches_current(receipt: Receipt, warmth: Warmth, current: tuple[PlanIdentity, PlanLane] | None) -> bool: - if current is None: - return False - entry, lane = current +def _runs_on(receipt: Receipt, lane: PlanLane) -> bool: return ( - entry.warmth is warmth - and lane.cell == receipt.cell + lane.cell == receipt.cell and lane.container_shape == receipt.container_shape and lane.concurrency == receipt.concurrency ) @@ -399,15 +397,21 @@ def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvi def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) - def hourly_cost(self, cell: Cell, shape: ContainerShape) -> float | None: - if _refusal_reason(cell, self.prices) is not None: - return None - return container_hourly_cost(self.prices[cell.machine_profile], shape, self.multiplier) + def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: + current = self.current.get(identity) + if current is None: + return False + entry, lane = current + return ( + entry.warmth is warmth + and lane.queue_pool == self.parameters.queue_pool + and lane.bundle == self.parameters.bundle_of(identity) + and _runs_on(receipt, lane) + ) def choose(self, identity: str) -> _Choice: level = self.levels[identity] requests = self.requests.get(identity, ()) - current = self.current.get(identity) options: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} receipts = self.receipts.get(identity, []) @@ -426,7 +430,7 @@ def choose(self, identity: str) -> _Choice: come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, blockers=warm, - current=_matches_current(receipt, Warmth.PINNED, current), + current=self.is_current(identity, receipt, Warmth.PINNED), ) ) cold_cost = hourly * self.window.cold_container_hours(requests, rate) @@ -458,7 +462,7 @@ def choose(self, identity: str) -> _Choice: come_alive_s=boot.seconds, basis=boot.basis, blockers=warm if blocker is None else (*warm, blocker), - current=_matches_current(receipt, warmth, current), + current=self.is_current(identity, receipt, warmth), ) ) feasible = [option for option in options if option.feasible] @@ -567,71 +571,131 @@ def next_cheaper(choice: _Choice) -> _Option | None: nearest = max(option.cost for option in cheaper) return min((option for option in cheaper if option.cost == nearest), key=lambda option: option.rank) - def lane_names(self, choices: Sequence[_Choice], reserved: set[str]) -> dict[str, str]: - bases: dict[str, str] = {} - for choice in choices: - if choice.chosen is not None: - profile = choice.chosen.receipt.cell.machine_profile - bases[choice.identity] = _slug(f"{choice.identity}-{profile}") or "lane" - counts = Counter(bases.values()) - return { - identity: base if counts[base] == 1 and base not in reserved else f"{base}-{_digest(identity)}" - for identity, base in bases.items() + @staticmethod + def lane_names(placed: Mapping[str, _Option], reserved: set[str]) -> dict[str, str]: + bases = { + identity: _slug(f"{identity}-{option.receipt.cell.machine_profile}") or "lane" + for identity, option in placed.items() } + counts = Counter(bases.values()) + taken = set(reserved) + names: dict[str, str] = {} + for identity, base in bases.items(): + name = base if counts[base] == 1 and base not in reserved else f"{base}-{_digest(identity)}" + if name in taken: + name = f"{base}-{_digest(identity, width=64)}" + if name in taken: + raise PlannerInputError(f"lane name {name!r} for {identity!r} is already taken") + taken.add(name) + names[identity] = name + return names + + def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: + receipt = option.receipt + return PlanLane( + name=name, + queue_pool=self.parameters.queue_pool, + bundle=self.parameters.bundle_of(identity), + cell=receipt.cell, + container_shape=receipt.container_shape, + concurrency=receipt.concurrency, + max_containers=self.parameters.max_containers, + envelope=envelope_for(option.warmth, self.parameters), + ) + + def within_ceiling(self, lane: PlanLane) -> PlanLane: + ceiling = self.parameters.max_containers + if lane.max_containers <= ceiling: + return lane + envelope = replace( + lane.envelope, + floor_min=min(lane.envelope.floor_min, ceiling), + floor_max=min(lane.envelope.floor_max, ceiling), + ) + return replace(lane, max_containers=ceiling, envelope=envelope) + + def slowest_receipt(self, identity: str, lane: PlanLane) -> Receipt | None: + matching = [(order, receipt) for order, receipt in self.receipts.get(identity, []) if _runs_on(receipt, lane)] + if not matching: + return None + return min(matching, key=lambda pair: (pair[1].throughput.requests_per_second, pair[0]))[1] + + def kept_blockers( + self, identity: str, entry: PlanIdentity, lane: PlanLane, receipt: Receipt | None + ) -> tuple[Blocker, ...]: + level = self.levels[identity] + blockers = [] if receipt is None else list(_warm_blockers(level, receipt)) + if entry.warmth in COLD_WARMTHS and level.come_alive_p95_s is not None: + boot = self.boots.get((identity, lane.cell, lane.container_shape, entry.warmth)) + if boot is not None and boot.seconds is not None: + blocker = _come_alive_blocker(level, boot.seconds) + blockers.extend(() if blocker is None else (blocker,)) + elif level.come_alive_p95_s == 0: + blockers.append(Blocker(field="comeAliveP95S", bound=0, measured=None)) + return tuple(blockers) + + def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) -> float | None: + requests = self.requests.get(identity, ()) + if not requests: + return 0.0 + if receipt is None: + return None + window = replace( + self.window, + ceiling=lane.max_containers, + active_floor=max(1, min(self.parameters.fallback_floor, lane.max_containers)), + ) + rate = receipt.throughput.requests_per_second + if lane.envelope.floor_min: + return window.above_floor_hours(requests, rate, lane.envelope.floor_min) + return window.cold_container_hours(requests, rate) def run(self) -> PlannerResult: choices = [self.choose(identity) for identity in sorted(self.levels)] + placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} kept = { choice.identity: self.current[choice.identity] for choice in choices if choice.chosen is None and choice.identity in self.current } - kept_lanes = {lane.name: lane for _, lane in kept.values()} - names = self.lane_names(choices, set(kept_lanes)) - lanes: dict[str, PlanLane] = dict(kept_lanes) - identities: list[PlanIdentity] = [entry for entry, _ in kept.values()] + staying = {identity: self.current[identity][1].name for identity, option in placed.items() if option.current} + reserved = {lane.name for _, lane in kept.values()} | set(staying.values()) + moving = {identity: option for identity, option in placed.items() if identity not in staying} + names = staying | self.lane_names(moving, reserved) + lanes = {lane.name: self.within_ceiling(lane) for _, lane in kept.values()} + for identity, option in placed.items(): + lanes[names[identity]] = self.placed_lane(names[identity], identity, option) + identities = [entry for entry, _ in kept.values()] + identities.extend( + PlanIdentity(identity=identity, lane=names[identity], warmth=option.warmth) + for identity, option in placed.items() + ) + new_plan = Plan( + placement=self.parameters.placement, + lanes=tuple(lanes[name] for name in sorted(lanes)), + identities=tuple(sorted(identities, key=lambda entry: entry.identity)), + ) + receipts: dict[str, Receipt | None] = {identity: option.receipt for identity, option in placed.items()} + receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) + lane_reports, costs = self._pricing(new_plan, receipts) reports: list[IdentityReport] = [] requests: list[MeasurementRequest] = [] levers: list[Lever] = [] - monthly_cost = 0.0 for choice in choices: requests.extend(self.receipt_requests(choice)) requests.extend(self.boot_requests(choice)) levers.extend(self.levers(choice)) - reports.append(self._identity_report(choice, names, kept)) - if choice.chosen is None: - continue - monthly_cost += choice.chosen.cost - receipt = choice.chosen.receipt - name = names[choice.identity] - lanes[name] = PlanLane( - name=name, - queue_pool=self.parameters.queue_pool, - bundle=self.parameters.bundle_of(choice.identity), - cell=receipt.cell, - container_shape=receipt.container_shape, - concurrency=receipt.concurrency, - max_containers=self.parameters.max_containers, - envelope=envelope_for(choice.chosen.warmth, self.parameters), + reports.append( + self._identity_report(choice, names, kept, costs.get(choice.identity), receipts.get(choice.identity)) ) - identities.append(PlanIdentity(identity=choice.identity, lane=name, warmth=choice.chosen.warmth)) - new_plan = Plan( - placement=self.parameters.placement, - lanes=tuple(lanes[name] for name in sorted(lanes)), - identities=tuple(sorted(identities, key=lambda entry: entry.identity)), - ) - lane_reports = self._lane_reports(new_plan) identity_changes, lane_changes = _changes(self.current_plan, new_plan) report = Report( placement=self.parameters.placement, currency=self.price_list.currency, parameters=self.parameters.to_dict(), - monthly_cost=monthly_cost, - envelope_max_monthly_cost=sum( - lane.envelope_max_monthly_cost for lane in lane_reports if lane.envelope_max_monthly_cost is not None - ), identities=tuple(reports), lanes=lane_reports, + shared_chart_lanes=_shared_chart_lanes(new_plan), measurement_requests=tuple(sorted(requests, key=_request_rank)), come_alive_levers=tuple(sorted(levers, key=_lever_rank)), queue_wait_unattained=tuple( @@ -644,7 +708,12 @@ def run(self) -> PlannerResult: return PlannerResult(plan=new_plan, report=report) def _identity_report( - self, choice: _Choice, names: Mapping[str, str], kept: Mapping[str, tuple[PlanIdentity, PlanLane]] + self, + choice: _Choice, + names: Mapping[str, str], + kept: Mapping[str, tuple[PlanIdentity, PlanLane]], + cost: float | None, + receipt: Receipt | None, ) -> IdentityReport: current = self.current.get(choice.identity) report = IdentityReport( @@ -658,8 +727,14 @@ def _identity_report( if chosen is None: reason = UnplacedReason.WARM_BOUND_MISSED if choice.has_valid_receipt else UnplacedReason.NO_RECEIPT if choice.identity in kept: + entry, lane = kept[choice.identity] return replace( - report, status=IdentityStatus.KEPT, lane=kept[choice.identity][0].lane, unplaced_reason=reason + report, + status=IdentityStatus.KEPT, + lane=entry.lane, + monthly_cost=cost, + unplaced_reason=reason, + blocked_by=self.kept_blockers(choice.identity, entry, lane, receipt), ) return replace(report, unplaced_reason=reason) nearest = self.next_cheaper(choice) @@ -668,29 +743,57 @@ def _identity_report( report, status=IdentityStatus.PLACED, lane=names[choice.identity], + monthly_cost=cost, chosen=chosen.summary(), peak_containers=max((_Window.needed(count, rate) for count in choice.requests), default=0), next_cheaper=None if nearest is None else nearest.summary(saving=chosen.cost - nearest.cost), ) - def _lane_reports(self, plan: Plan) -> tuple[LaneReport, ...]: + def _pricing( + self, plan: Plan, receipts: Mapping[str, Receipt | None] + ) -> tuple[tuple[LaneReport, ...], dict[str, float]]: residents: defaultdict[str, list[str]] = defaultdict(list) for entry in plan.identities: residents[entry.lane].append(entry.identity) - reports = [] + hours = self.parameters.hours_per_month + reports: list[LaneReport] = [] + costs: dict[str, float] = {} for lane in plan.lanes: - hourly = self.hourly_cost(lane.cell, lane.container_shape) - hours = self.parameters.hours_per_month + hosted = tuple(sorted(residents[lane.name])) + reason = _refusal_reason(lane.cell, self.prices) + if reason is not None: + reports.append(LaneReport(name=lane.name, identities=hosted, unpriced_reason=reason)) + continue + hourly = container_hourly_cost( + self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier + ) + floor = lane.envelope.floor_min * hours + demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} + priced = {identity: value for identity, value in demand.items() if value is not None} + costs.update((identity, hourly * (floor + value)) for identity, value in priced.items()) reports.append( LaneReport( name=lane.name, - identities=tuple(sorted(residents[lane.name])), + identities=hosted, hourly_cost=hourly, - floor_monthly_cost=None if hourly is None else hourly * lane.envelope.floor_min * hours, - envelope_max_monthly_cost=None if hourly is None else hourly * lane.envelope.floor_max * hours, + monthly_cost=hourly * (floor + sum(priced.values())), + floor_monthly_cost=hourly * lane.envelope.floor_min * hours, + envelope_max_monthly_cost=hourly * lane.envelope.floor_max * hours, + unpriced_identities=tuple(identity for identity in hosted if identity not in priced), ) ) - return tuple(reports) + return tuple(reports), costs + + +def _shared_chart_lanes(plan: Plan) -> tuple[SharedChartLane, ...]: + grouped: defaultdict[tuple[str, str, str], list[str]] = defaultdict(list) + for lane in plan.lanes: + grouped[(lane.queue_pool, lane.cell.machine_profile, lane.bundle)].append(lane.name) + return tuple( + SharedChartLane(queue_pool=pool, machine_profile=profile, bundle=bundle, lanes=tuple(sorted(names))) + for (pool, profile, bundle), names in sorted(grouped.items()) + if len(names) > 1 + ) def _changes(current: Plan | None, new: Plan) -> tuple[tuple[IdentityChange, ...], tuple[LaneChange, ...]]: diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 462bbe3ed..f285f71bc 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -70,10 +70,10 @@ def _compact(fields: dict[str, object | None]) -> dict[str, object]: class Blocker: field: str bound: int - measured: float + measured: float | None def to_dict(self) -> dict[str, object]: - return {"field": self.field, "bound": self.bound, "measured": self.measured} + return _compact({"field": self.field, "bound": self.bound, "measured": self.measured}) @dataclass(frozen=True, slots=True, kw_only=True) @@ -113,11 +113,13 @@ class IdentityReport: service_level: ServiceLevel demand_requests: int lane: str | None = None + monthly_cost: float | None = None chosen: Option | None = None peak_containers: int | None = None next_cheaper: Option | None = None current: PlanIdentity | None = None unplaced_reason: UnplacedReason | None = None + blocked_by: tuple[Blocker, ...] = () def to_dict(self) -> dict[str, object]: return _compact( @@ -127,6 +129,8 @@ def to_dict(self) -> dict[str, object]: "serviceLevel": self.service_level.to_dict(), "demandRequests": self.demand_requests, "lane": self.lane, + "monthlyCost": None if self.monthly_cost is None else _money(self.monthly_cost), + "blockedBy": [blocker.to_dict() for blocker in self.blocked_by] or None, "chosen": None if self.chosen is None else self.chosen.to_dict(), "peakContainers": self.peak_containers, "nextCheaper": None if self.next_cheaper is None else self.next_cheaper.to_dict(), @@ -140,9 +144,16 @@ def to_dict(self) -> dict[str, object]: class LaneReport: name: str identities: tuple[str, ...] - hourly_cost: float | None - floor_monthly_cost: float | None - envelope_max_monthly_cost: float | None + hourly_cost: float | None = None + monthly_cost: float | None = None + floor_monthly_cost: float | None = None + envelope_max_monthly_cost: float | None = None + unpriced_reason: str | None = None + unpriced_identities: tuple[str, ...] = () + + @property + def unpriced(self) -> bool: + return self.unpriced_reason is not None or bool(self.unpriced_identities) def to_dict(self) -> dict[str, object]: return _compact( @@ -150,14 +161,33 @@ def to_dict(self) -> dict[str, object]: "name": self.name, "identities": list(self.identities), "hourlyCost": None if self.hourly_cost is None else _rate(self.hourly_cost), + "monthlyCost": None if self.monthly_cost is None else _money(self.monthly_cost), "floorMonthlyCost": None if self.floor_monthly_cost is None else _money(self.floor_monthly_cost), "envelopeMaxMonthlyCost": ( None if self.envelope_max_monthly_cost is None else _money(self.envelope_max_monthly_cost) ), + "unpricedReason": self.unpriced_reason, + "unpricedIdentities": list(self.unpriced_identities) or None, } ) +@dataclass(frozen=True, slots=True, kw_only=True) +class SharedChartLane: + queue_pool: str + machine_profile: str + bundle: str + lanes: tuple[str, ...] + + def to_dict(self) -> dict[str, object]: + return { + "queuePool": self.queue_pool, + "machineProfile": self.machine_profile, + "bundle": self.bundle, + "lanes": list(self.lanes), + } + + @dataclass(frozen=True, slots=True, kw_only=True) class MeasurementRequest: identity: str @@ -269,10 +299,9 @@ class Report: placement: str currency: str parameters: Mapping[str, object] - monthly_cost: float - envelope_max_monthly_cost: float identities: tuple[IdentityReport, ...] lanes: tuple[LaneReport, ...] + shared_chart_lanes: tuple[SharedChartLane, ...] measurement_requests: tuple[MeasurementRequest, ...] come_alive_levers: tuple[Lever, ...] queue_wait_unattained: tuple[str, ...] @@ -280,6 +309,18 @@ class Report: identity_changes: tuple[IdentityChange, ...] lane_changes: tuple[LaneChange, ...] + @property + def monthly_cost(self) -> float: + return sum(lane.monthly_cost for lane in self.lanes if lane.monthly_cost is not None) + + @property + def envelope_max_monthly_cost(self) -> float: + return sum(lane.envelope_max_monthly_cost for lane in self.lanes if lane.envelope_max_monthly_cost is not None) + + @property + def unpriced_lanes(self) -> tuple[str, ...]: + return tuple(lane.name for lane in self.lanes if lane.unpriced) + @property def unchanged(self) -> bool: return not self.identity_changes and not self.lane_changes @@ -300,9 +341,11 @@ def to_dict(self) -> dict[str, object]: "totals": { "monthlyCost": _money(self.monthly_cost), "envelopeMaxMonthlyCost": _money(self.envelope_max_monthly_cost), + "unpricedLanes": list(self.unpriced_lanes), }, "identities": [entry.to_dict() for entry in self.identities], "lanes": [lane.to_dict() for lane in self.lanes], + "sharedChartLanes": [shared.to_dict() for shared in self.shared_chart_lanes], "measurementRequests": [request.to_dict() for request in self.measurement_requests], "comeAliveLevers": [lever.to_dict() for lever in self.come_alive_levers], "queueWaitUnattained": list(self.queue_wait_unattained), diff --git a/packages/sie_placement/tests/fixtures/planner/expected-plan.json b/packages/sie_placement/tests/fixtures/planner/expected-plan.json index 1edfd3444..303a4ed79 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-plan.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-plan.json @@ -3,26 +3,6 @@ "schemaVersion": 1, "placement": "us-central1", "lanes": [ - { - "name": "baai-bge-m3-l4", - "queuePool": "default", - "bundle": "default", - "cell": { - "machineProfile": "l4", - "gpuProduct": "NVIDIA-L4" - }, - "containerShape": { - "gpuCount": 1, - "cpuCores": 4, - "memoryMiB": 16384 - }, - "concurrency": 16, - "maxContainers": 4, - "envelope": { - "floorMin": 1, - "floorMax": 2 - } - }, { "name": "baai-bge-m3-sparse-l4", "queuePool": "default", @@ -107,7 +87,7 @@ "identities": [ { "identity": "BAAI/bge-m3", - "lane": "baai-bge-m3-l4", + "lane": "default-l4-default", "warmth": "pinned" }, { diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index ee6424bc9..7de1fe3db 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -18,7 +18,10 @@ }, "totals": { "monthlyCost": 1947.88, - "envelopeMaxMonthlyCost": 10658.0 + "envelopeMaxMonthlyCost": 9139.6, + "unpricedLanes": [ + "default-l4-default" + ] }, "identities": [ { @@ -30,7 +33,8 @@ "queueWaitP99Ms": 500 }, "demandRequests": 180000, - "lane": "baai-bge-m3-l4", + "lane": "default-l4-default", + "monthlyCost": 759.2, "chosen": { "cell": { "machineProfile": "l4", @@ -96,6 +100,7 @@ }, "demandRequests": 3, "lane": "baai-bge-m3-sparse-l4", + "monthlyCost": 47.45, "chosen": { "cell": { "machineProfile": "l4", @@ -128,6 +133,7 @@ }, "demandRequests": 42, "lane": "qwen-qwen3-8b-a100-80gb", + "monthlyCost": 382.03, "chosen": { "cell": { "machineProfile": "a100-80gb", @@ -192,6 +198,7 @@ }, "demandRequests": 3600, "lane": "qwen-qwen3-embedding-4b-l4", + "monthlyCost": 759.2, "chosen": { "cell": { "machineProfile": "l4", @@ -271,32 +278,29 @@ } ], "lanes": [ - { - "name": "baai-bge-m3-l4", - "identities": [ - "BAAI/bge-m3" - ], - "hourlyCost": 1.04, - "floorMonthlyCost": 759.2, - "envelopeMaxMonthlyCost": 1518.4 - }, { "name": "baai-bge-m3-sparse-l4", "identities": [ "BAAI/bge-m3:sparse" ], "hourlyCost": 1.04, + "monthlyCost": 47.45, "floorMonthlyCost": 0.0, "envelopeMaxMonthlyCost": 1518.4 }, { "name": "default-l4-default", "identities": [ + "BAAI/bge-m3", "intfloat/e5-base-v2" ], "hourlyCost": 1.04, + "monthlyCost": 759.2, "floorMonthlyCost": 759.2, - "envelopeMaxMonthlyCost": 1518.4 + "envelopeMaxMonthlyCost": 1518.4, + "unpricedIdentities": [ + "intfloat/e5-base-v2" + ] }, { "name": "qwen-qwen3-8b-a100-80gb", @@ -304,6 +308,7 @@ "Qwen/Qwen3-8B" ], "hourlyCost": 3.14, + "monthlyCost": 382.03, "floorMonthlyCost": 0.0, "envelopeMaxMonthlyCost": 4584.4 }, @@ -313,10 +318,23 @@ "Qwen/Qwen3-Embedding-4B" ], "hourlyCost": 1.04, + "monthlyCost": 759.2, "floorMonthlyCost": 759.2, "envelopeMaxMonthlyCost": 1518.4 } ], + "sharedChartLanes": [ + { + "queuePool": "default", + "machineProfile": "l4", + "bundle": "default", + "lanes": [ + "baai-bge-m3-sparse-l4", + "default-l4-default", + "qwen-qwen3-embedding-4b-l4" + ] + } + ], "measurementRequests": [ { "identity": "Qwen/Qwen3-Reranker-0.6B", @@ -593,19 +611,6 @@ ], "changes": { "identities": [ - { - "identity": "BAAI/bge-m3", - "before": { - "identity": "BAAI/bge-m3", - "lane": "default-l4-default", - "warmth": "pinned" - }, - "after": { - "identity": "BAAI/bge-m3", - "lane": "baai-bge-m3-l4", - "warmth": "pinned" - } - }, { "identity": "BAAI/bge-m3:sparse", "after": { @@ -624,29 +629,6 @@ } ], "lanes": [ - { - "name": "baai-bge-m3-l4", - "after": { - "name": "baai-bge-m3-l4", - "queuePool": "default", - "bundle": "default", - "cell": { - "machineProfile": "l4", - "gpuProduct": "NVIDIA-L4" - }, - "containerShape": { - "gpuCount": 1, - "cpuCores": 4, - "memoryMiB": 16384 - }, - "concurrency": 16, - "maxContainers": 4, - "envelope": { - "floorMin": 1, - "floorMax": 2 - } - } - }, { "name": "baai-bge-m3-sparse-l4", "after": { diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index 27e8a02df..9d762a98d 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -55,14 +55,14 @@ L4_HOURLY = 1.0 + 4 * 0.1 -def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8): +def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10.0): return Receipt( identity=identity, cell=cell, container_shape=shape, concurrency=concurrency, request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), - throughput=Throughput(requests_per_second=10.0), + throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}"), ) @@ -81,7 +81,7 @@ def boot(identity, seconds, warmth=Warmth.COLD_WITH_BOUND, cell=L4, *, shape=SHA ) -def busy(*identities, hours=6): +def busy(*identities, hours=6, requests=10): return Demand( start=START, end=START + timedelta(hours=24), @@ -89,7 +89,9 @@ def busy(*identities, hours=6): DemandSeries( identity=identity, operation=Operation.ENCODE, - hours=tuple(DemandHour(start=START + timedelta(hours=hour), requests=10) for hour in range(hours)), + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=requests) for hour in range(hours) + ), ) for identity in identities ), @@ -213,3 +215,17 @@ def test_each_cell_shape_and_warmth_is_one_lever_at_its_cheapest_receipt(): assert by_key[(SHAPE, Warmth.COLD_WITH_BOUND)].monthly_cost == pytest.approx(cold_cost()) assert by_key[(WIDE, Warmth.COLD_WITH_BOUND)].monthly_cost == pytest.approx((1.0 + 8 * 0.1) * 6 * 2 * SCALE) assert by_key[(SHAPE, Warmth.SNAPSHOT_COLD)].permitted_bound_s == 90 + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_a_lever_is_priced_at_the_cheapest_receipt_of_its_cell_and_shape(reverse): + receipts = [receipt("org/model", rps=0.01), receipt("org/model", concurrency=16, rps=0.02)] + report = run( + {"org/model": 60}, + receipts[::-1] if reverse else receipts, + [boot("org/model", 240.0)], + busy("org/model", requests=1), + ) + (lever,) = report.come_alive_levers + assert lever.monthly_cost == pytest.approx(L4_HOURLY * 6 * (1 / 0.02 + 1 * 2 * 900) / 3600 * SCALE) + assert lever.saving_under_bound == pytest.approx(L4_HOURLY * MONTH - lever.monthly_cost) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 90776c2bd..c039599a0 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -191,7 +191,17 @@ def run(service_levels, receipts=(), boots=(), *, prices=None, hourly=None, curr ) -def single_lane_plan(name="shared", warmth=Warmth.PINNED, cell=L4, identity=IDENTITY) -> Plan: +def single_lane_plan( + name="shared", + warmth=Warmth.PINNED, + cell=L4, + identity=IDENTITY, + *, + others=(), + concurrency=8, + max_containers=2, + envelope=None, +) -> Plan: floor = 1 if warmth is Warmth.PINNED else 0 return Plan( placement="test", @@ -202,12 +212,14 @@ def single_lane_plan(name="shared", warmth=Warmth.PINNED, cell=L4, identity=IDEN bundle="default", cell=cell, container_shape=SHAPE, - concurrency=8, - max_containers=2, - envelope=Envelope(floor_min=floor, floor_max=max(floor, 1)), + concurrency=concurrency, + max_containers=max_containers, + envelope=envelope or Envelope(floor_min=floor, floor_max=max(floor, 1)), ), ), - identities=(PlanIdentity(identity=identity, lane=name, warmth=warmth),), + identities=tuple( + PlanIdentity(identity=resident, lane=name, warmth=warmth) for resident in sorted((identity, *others)) + ), ) @@ -250,6 +262,23 @@ def test_a_pinned_cell_costs_its_floor_plus_the_hours_demand_needs_above_it(): assert report.demand_requests == 9100 +def test_demand_sums_its_operations_in_each_hour(): + hour = START + timedelta(hours=3) + mixed = Demand( + start=START, + end=START + timedelta(hours=WINDOW_HOURS), + series=tuple( + DemandSeries(identity=IDENTITY, operation=operation, hours=(DemandHour(start=hour, requests=requests),)) + for operation, requests in ((Operation.ENCODE, 5000), (Operation.SCORE, 4000)) + ), + ) + result = plan(level(come_alive_p95_s=0), [receipt(rps=1.0)], (), L4_ONLY, mixed, None, parameters()) + report = result.report.identity(IDENTITY) + assert report.demand_requests == 9000 + assert report.peak_containers == 3 + assert report.chosen.monthly_cost == pytest.approx(MONTH + (3 - 1) * SCALE) + + def test_needed_containers_stop_at_the_reviewed_ceiling(): result = run( level(come_alive_p95_s=0), [receipt(rps=1.0)], prices=L4_ONLY, hourly={IDENTITY: {0: 36000}}, max_containers=2 @@ -271,6 +300,20 @@ def test_a_cold_cell_costs_its_work_and_the_tail_each_burst_holds(): assert chosen.monthly_cost == pytest.approx((one_burst + capped_hour) / 3600 * SCALE) +def test_a_saturated_cold_hour_runs_at_most_the_ceiling(): + result = run( + level(come_alive_p95_s=120), + [receipt(rps=1.0)], + [boot()], + prices=L4_ONLY, + hourly={IDENTITY: {0: 36000}}, + max_containers=2, + ) + chosen = result.report.identity(IDENTITY).chosen + assert chosen.warmth is Warmth.COLD_WITH_BOUND + assert chosen.monthly_cost == pytest.approx(2 * SCALE) + + def test_an_active_lane_without_a_fallback_floor_holds_one_container(): result = run( level(come_alive_p95_s=120), @@ -321,6 +364,28 @@ def test_a_cold_posture_without_boot_evidence_is_a_measurement_request(): assert request.saving_bound == pytest.approx(MONTH) +@pytest.mark.parametrize("reverse", [False, True]) +def test_a_cell_without_boot_evidence_asks_once_at_its_cheapest_receipt(reverse): + receipts = [receipt(rps=0.01), receipt(concurrency=16, rps=0.02)] + result = run( + level(come_alive_p95_s=120), + receipts[::-1] if reverse else receipts, + prices=L4_ONLY, + hourly={IDENTITY: {0: 1}}, + ) + (request,) = result.report.measurement_requests + assert request.measure == (Measurement.BOOT_EVIDENCE,) + assert request.cost_bound == pytest.approx((1 / 0.02 + 1 * 2 * 900) / 3600 * SCALE) + + +def test_snapshot_restores_are_never_requested(): + result = run(level(come_alive_p95_s=120), [receipt()], [boot(seconds=300.0)], prices=L4_ONLY) + report = result.report + assert report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + assert report.measurement_requests == () + assert [lever.warmth for lever in report.come_alive_levers] == [Warmth.COLD_WITH_BOUND] + + def test_a_failed_boot_is_not_selectable_and_asks_for_a_new_measurement(): boots = [boot(seconds=900.0, outcome=BootOutcome.TIMED_OUT)] result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) @@ -344,6 +409,14 @@ def test_the_slowest_successful_boot_stands_for_the_percentile(): assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=120, measured=130.0),) +def test_boot_seconds_render_the_same_whatever_their_number_type(): + boots = [boot(seconds=60, uri="boots/1.json"), boot(seconds=60.0, uri="boots/2.json")] + first = run(level(come_alive_p95_s=120), [receipt()], boots, prices=L4_ONLY) + second = run(level(come_alive_p95_s=120), [receipt()], boots[::-1], prices=L4_ONLY) + assert rendered(first) == rendered(second) + assert first.report.to_dict()["identities"][0]["chosen"]["comeAliveS"] == 60.0 + + def test_own_boot_evidence_replaces_borrowed_evidence(): borrowed = boot(seconds=150.0, borrowed=True, uri="boots/borrowed.json") own = boot(seconds=60.0, uri="boots/own.json") @@ -581,6 +654,9 @@ def test_the_plan_carries_the_envelope_and_the_report_prices_its_maximum(): assert lane_report.envelope_max_monthly_cost == pytest.approx(3 * MONTH) assert result.report.envelope_max_monthly_cost == pytest.approx(3 * MONTH) assert result.report.monthly_cost == pytest.approx(MONTH) + assert lane_report.monthly_cost == pytest.approx(MONTH) + assert result.report.identity(IDENTITY).monthly_cost == pytest.approx(MONTH) + assert result.report.unpriced_lanes == () def test_every_declared_queue_wait_bound_is_reported_unattained(): @@ -604,6 +680,114 @@ def test_an_identity_without_a_feasible_cell_keeps_its_current_lane(): assert report.current == current.identities[0] assert result.report.unchanged assert result.report.lanes[0].floor_monthly_cost == pytest.approx(MONTH) + assert report.monthly_cost == pytest.approx(MONTH) + assert report.blocked_by == (Blocker(field="warmP99Ms", bound=50, measured=100.0),) + assert result.report.monthly_cost == pytest.approx(MONTH) + assert result.report.to_dict()["totals"] == { + "monthlyCost": MONTH, + "envelopeMaxMonthlyCost": MONTH, + "unpricedLanes": [], + } + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_a_kept_lane_prices_its_demand_at_the_slowest_receipt_for_its_cell(reverse): + current = single_lane_plan(max_containers=4, envelope=Envelope(floor_min=1, floor_max=2)) + receipts = [receipt(rps=1.0, p99=100.0, uri="receipts/slow"), receipt(rps=2.0, p99=100.0, uri="receipts/fast")] + result = run( + level(come_alive_p95_s=0, warm_p99_ms=50), + receipts[::-1] if reverse else receipts, + prices=L4_ONLY, + hourly={IDENTITY: {3: 9000}}, + current=current, + ) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert report.status is IdentityStatus.KEPT + assert report.monthly_cost == pytest.approx(MONTH + (3 - 1) * SCALE) + (lane,) = result.report.lanes + assert lane.monthly_cost == pytest.approx(MONTH + (3 - 1) * SCALE) + assert result.report.monthly_cost == pytest.approx(MONTH + (3 - 1) * SCALE) + assert result.report.envelope_max_monthly_cost == pytest.approx(2 * MONTH) + + +@pytest.mark.parametrize( + ("warmth", "ceiling", "hourly", "hours"), + [ + (Warmth.PINNED, 2, {3: 9000}, MONTH + (2 - 1) * SCALE), + (Warmth.COLD_WITH_BOUND, 1, {0: 1}, (1 / 1.0 + 1 * 1 * 900) / 3600 * SCALE), + ], +) +def test_a_kept_lane_prices_its_demand_under_its_own_ceiling(warmth, ceiling, hourly, hours): + current = single_lane_plan(warmth=warmth, max_containers=ceiling) + result = run( + level(come_alive_p95_s=600, warm_p99_ms=50), + [receipt(rps=1.0, p99=100.0)], + prices=L4_ONLY, + hourly={IDENTITY: hourly}, + current=current, + ) + assert result.plan == current + assert result.report.identity(IDENTITY).monthly_cost == pytest.approx(hours) + + +def test_a_kept_lane_without_a_receipt_for_its_demand_lists_it_unpriced(): + current = single_lane_plan() + result = run(level(come_alive_p95_s=0), [], prices=L4_ONLY, hourly={IDENTITY: {0: 10}}, current=current) + report = result.report.identity(IDENTITY) + assert (report.status, report.monthly_cost) == (IdentityStatus.KEPT, None) + (lane,) = result.report.lanes + assert lane.unpriced_identities == (IDENTITY,) + assert lane.monthly_cost == pytest.approx(MONTH) + assert result.report.unpriced_lanes == ("shared",) + assert result.report.monthly_cost == pytest.approx(MONTH) + + +def test_a_kept_lane_on_a_refused_cell_is_kept_and_reported_unpriced(): + other = Cell(machine_profile="l4", gpu_product="NVIDIA-A10G") + current = single_lane_plan(cell=other) + result = run(level(come_alive_p95_s=0), [], prices=L4_ONLY, current=current) + assert result.plan == current + (lane,) = result.report.lanes + assert lane.unpriced_reason == ( + "realized GPU product 'NVIDIA-A10G' differs from 'NVIDIA-L4', the product machine profile 'l4' requests" + ) + assert (lane.hourly_cost, lane.monthly_cost, lane.envelope_max_monthly_cost) == (None, None, None) + assert result.report.identity(IDENTITY).monthly_cost is None + assert result.report.to_dict()["totals"] == { + "monthlyCost": 0.0, + "envelopeMaxMonthlyCost": 0.0, + "unpricedLanes": ["shared"], + } + + +def test_a_kept_lane_never_exceeds_the_reviewed_ceiling(): + current = single_lane_plan(max_containers=4, envelope=Envelope(floor_min=2, floor_max=3)) + result = run(level(come_alive_p95_s=0), [], prices=L4_ONLY, current=current, max_containers=1) + (lane,) = result.plan.lanes + assert (lane.max_containers, lane.envelope) == (1, Envelope(floor_min=1, floor_max=1)) + (change,) = result.report.lane_changes + assert (change.name, change.before, change.after) == ("shared", current.lanes[0], lane) + + +@pytest.mark.parametrize( + ("bound", "boots", "expected"), + [ + (0, (), (Blocker(field="comeAliveP95S", bound=0, measured=None),)), + (120, (), ()), + (120, (boot(seconds=200.0),), (Blocker(field="comeAliveP95S", bound=120, measured=200.0),)), + (300, (boot(seconds=200.0),), ()), + ], +) +def test_a_kept_cold_placement_names_the_come_alive_bound_it_is_known_to_miss(bound, boots, expected): + current = single_lane_plan(warmth=Warmth.COLD_WITH_BOUND) + result = run(level(come_alive_p95_s=bound), [], boots, prices=L4_ONLY, current=current) + report = result.report.identity(IDENTITY) + assert report.status is IdentityStatus.KEPT + assert report.blocked_by == expected + assert report.monthly_cost == 0 + if expected and expected[0].measured is None: + assert report.to_dict()["blockedBy"] == [{"field": "comeAliveP95S", "bound": 0}] def test_an_identity_without_a_feasible_cell_or_a_current_lane_is_left_out(): @@ -639,6 +823,141 @@ def test_a_derived_lane_name_never_takes_a_kept_lane_name(): assert names[IDENTITY] == f"org-model-l4-{hashlib.sha256(IDENTITY.encode()).hexdigest()[:8]}" +def test_a_current_placement_keeps_its_lane_name(): + current = single_lane_plan(name="legacy") + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current, max_containers=2) + assert result.plan == current + assert result.report.identity(IDENTITY).lane == "legacy" + assert result.report.unchanged + + +def test_a_current_placement_that_differs_only_in_its_envelope_keeps_its_lane_name(): + current = single_lane_plan(name="legacy", max_containers=8, envelope=Envelope(floor_min=2, floor_max=5)) + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current) + (lane,) = result.plan.lanes + assert (lane.name, lane.max_containers, lane.envelope) == ("legacy", 4, Envelope(floor_min=1, floor_max=1)) + assert result.report.identity_changes == () + assert [change.name for change in result.report.lane_changes] == ["legacy"] + + +def test_a_current_lane_that_differs_in_concurrency_is_not_the_current_placement(): + current = single_lane_plan(name="legacy", max_containers=4) + receipts = [receipt(concurrency=16), receipt(concurrency=8)] + result = run(level(come_alive_p95_s=0), receipts, prices=L4_ONLY, current=current) + (lane,) = result.plan.lanes + assert (lane.name, lane.concurrency) == ("legacy", 8) + assert result.report.unchanged + moved = run(level(come_alive_p95_s=0), [receipts[0]], prices=L4_ONLY, current=current) + (lane,) = moved.plan.lanes + assert (lane.name, lane.concurrency) == ("org-model-l4", 16) + assert [(change.name, change.before is None) for change in moved.report.lane_changes] == [ + ("legacy", False), + ("org-model-l4", True), + ] + + +def test_a_current_placement_on_a_shared_lane_stays_beside_its_kept_neighbour(): + current = single_lane_plan(others=("org/kept",), max_containers=4) + service_levels = levels( + (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) + ) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current, hourly={IDENTITY: {3: 300000}}) + assert result.plan == current + assert result.report.unchanged + (lane,) = result.report.lanes + assert lane.identities == ("org/kept", IDENTITY) + above = (min(4, math.ceil(300000 / 36000)) - 1) * SCALE + assert lane.monthly_cost == pytest.approx(MONTH + above) + assert result.report.identity(IDENTITY).monthly_cost == pytest.approx(MONTH + above) + assert result.report.identity("org/kept").monthly_cost == pytest.approx(MONTH) + assert result.report.monthly_cost == pytest.approx(MONTH + above) + + +def test_current_placements_that_share_a_lane_keep_sharing_it(): + current = single_lane_plan(others=("org/other",), max_containers=4) + service_levels = levels((IDENTITY, ServiceLevel()), ("org/other", ServiceLevel())) + result = run(service_levels, [receipt(), receipt(identity="org/other")], prices=L4_ONLY, current=current) + assert result.plan == current + assert result.report.unchanged + assert result.report.monthly_cost == pytest.approx(MONTH) + assert result.report.shared_chart_lanes == () + + +def test_a_placement_that_moves_leaves_its_shared_lane_to_the_kept_neighbour(): + current = single_lane_plan(others=("org/kept",), max_containers=4) + service_levels = levels( + (IDENTITY, ServiceLevel(come_alive_p95_s=0, warm_p99_ms=250)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) + ) + receipts = [receipt(p99=300.0), receipt(cell=A100)] + result = run(service_levels, receipts, current=current) + names = {entry.identity: entry.lane for entry in result.plan.identities} + assert names == {IDENTITY: "org-model-a100-80gb", "org/kept": "shared"} + assert result.report.identity(IDENTITY).current == current.identities[1] + + +@pytest.mark.parametrize( + ("overrides", "moved"), + [({"bundles": {IDENTITY: "sglang"}}, ("default", "sglang")), ({"queue_pool": "batch"}, ("batch", "default"))], +) +def test_a_new_queue_pool_or_bundle_moves_the_identity_off_its_current_lane(overrides, moved): + current = single_lane_plan(others=("org/kept",), max_containers=4) + service_levels = levels( + (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) + ) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current, **overrides) + lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} + assert lanes == {"org-model-l4": moved, "shared": ("default", "default")} + + +def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): + service_levels = levels(("org/model", ServiceLevel()), ("org-model", ServiceLevel())) + receipts = [receipt(identity="org/model"), receipt(identity="org-model")] + first = run(service_levels, receipts, prices=L4_ONLY) + second = run(levels(("org/model", ServiceLevel())), receipts, prices=L4_ONLY, current=first.plan) + digest = hashlib.sha256(b"org-model").hexdigest()[:8] + kept_name = f"org-model-l4-{hashlib.sha256(b'org/model').hexdigest()[:8]}" + assert [entry.lane for entry in second.plan.identities] == [kept_name] + assert [(change.identity, change.after) for change in second.report.identity_changes] == [("org-model", None)] + assert [(change.name, change.after) for change in second.report.lane_changes] == [(f"org-model-l4-{digest}", None)] + + +def test_plan_lanes_that_name_one_chart_lane_are_reported(): + service_levels = levels(("org/a", ServiceLevel()), ("org/b", ServiceLevel()), ("org/c", ServiceLevel())) + receipts = [receipt(identity=identity) for identity in ("org/a", "org/b", "org/c")] + result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/c": "sglang"}) + (shared,) = result.report.shared_chart_lanes + assert shared.to_dict() == { + "queuePool": "default", + "machineProfile": "l4", + "bundle": "default", + "lanes": ["org-a-l4", "org-b-l4"], + } + apart = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "a", "org/b": "b"}) + assert apart.report.shared_chart_lanes == () + + +def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): + digest = hashlib.sha256(IDENTITY.encode()).hexdigest() + first = single_lane_plan(name="org-model-l4", identity="org/kept") + second = single_lane_plan(name=f"org-model-l4-{digest[:8]}", identity="org/kept-too") + current = Plan( + placement="test", lanes=(*first.lanes, *second.lanes), identities=(*first.identities, *second.identities) + ) + service_levels = levels( + (IDENTITY, ServiceLevel()), + ("org/kept", ServiceLevel(come_alive_p95_s=0)), + ("org/kept-too", ServiceLevel(come_alive_p95_s=0)), + ) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current) + names = {entry.identity: entry.lane for entry in result.plan.identities} + assert names == { + IDENTITY: f"org-model-l4-{digest}", + "org/kept": "org-model-l4", + "org/kept-too": f"org-model-l4-{digest[:8]}", + } + assert {lane.name: lane for lane in result.plan.lanes}[f"org-model-l4-{digest[:8]}"] == second.lanes[0] + + def test_bundles_and_the_queue_pool_come_from_the_parameters(): service_levels = levels(("org/embed", ServiceLevel()), ("org/generate", ServiceLevel())) result = run( @@ -781,7 +1100,7 @@ def test_the_estate_places_every_identity_on_a_measured_feasible_cell(): def test_the_estate_report_states_its_findings(): report = plan(**estate()).report dense = report.identity("BAAI/bge-m3") - assert (dense.chosen.cell, dense.chosen.warmth) == (L4, Warmth.PINNED) + assert (dense.chosen.cell, dense.chosen.warmth, dense.lane) == (L4, Warmth.PINNED, "default-l4-default") assert dense.chosen.monthly_cost == pytest.approx(1.04 * MONTH) assert dense.next_cheaper.warmth is Warmth.SNAPSHOT_COLD assert dense.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=0, measured=8.5),) @@ -793,16 +1112,22 @@ def test_the_estate_report_states_its_findings(): assert generation.chosen.monthly_cost == pytest.approx(3.14 * 2 * 2 * SCALE) assert generation.next_cheaper.blocked_by == (Blocker(field="itlP99Ms", bound=80, measured=118.0),) assert report.identity("Qwen/Qwen3-Reranker-0.6B").status is IdentityStatus.UNPLACED - assert report.identity("intfloat/e5-base-v2").status is IdentityStatus.KEPT + kept = report.identity("intfloat/e5-base-v2") + assert (kept.status, kept.lane, kept.monthly_cost) == (IdentityStatus.KEPT, "default-l4-default", None) assert [refusal.record for refusal in report.refusals] == [RefusedRecord.BOOT_EVIDENCE, RefusedRecord.RECEIPT] assert report.queue_wait_unattained == ("BAAI/bge-m3", "BAAI/bge-m3:sparse", "intfloat/e5-base-v2") - assert [change.identity for change in report.identity_changes] == [ - "BAAI/bge-m3", - "BAAI/bge-m3:sparse", - "Qwen/Qwen3-Embedding-4B", - ] - assert report.monthly_cost == pytest.approx( - sum(entry.chosen.monthly_cost for entry in report.identities if entry.chosen is not None) + assert [change.identity for change in report.identity_changes] == ["BAAI/bge-m3:sparse", "Qwen/Qwen3-Embedding-4B"] + assert report.monthly_cost == pytest.approx(sum(lane.monthly_cost for lane in report.lanes)) + shared = {lane.name: lane for lane in report.lanes}["default-l4-default"] + assert shared.identities == ("BAAI/bge-m3", "intfloat/e5-base-v2") + assert shared.monthly_cost == pytest.approx(dense.monthly_cost) + assert shared.unpriced_identities == ("intfloat/e5-base-v2",) + assert report.unpriced_lanes == ("default-l4-default",) + (chart_lane,) = report.shared_chart_lanes + assert chart_lane.lanes == ("baai-bge-m3-sparse-l4", "default-l4-default", "qwen-qwen3-embedding-4b-l4") + current = estate()["current_plan"] + assert sum(lane.envelope.floor_min for lane in plan(**estate()).plan.lanes) == 1 + sum( + lane.envelope.floor_min for lane in current.lanes ) From e80312653c97dc4edd457b6816b45cce79107a8c Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 03:12:43 +0100 Subject: [PATCH 03/40] feat(placement): emit sie-cluster Helm values from a plan helm_values turns a placement plan into a values overlay for the sie-cluster chart: one worker group per machine profile, whose bundle lanes carry minReplicas (the envelope's floorMin) and maxReplicas (maxContainers), and one static queue pool per named queue pool, which carries the lanes' floor as minimumWorkerCount and their pinned identities as pinnedModels. Lane tuples are canonicalized as the chart's lane-token helper does. It refuses a plan the chart and the gateway would not realize as planned: a zero-floor lane whose floorMax falls below the chart's fallback floor, more lanes than the chart's release limit (192, or 96 when the release installs KEDA or Prometheus), plan lanes on one chart lane or worker group that disagree on what it carries, pins on the default queue pool, which has no static queue pool, and lanes on one static queue pool that need different floors, pins or bundles. A golden test pins the emitted overlay, and chart render tests render it with helm template to check the lanes, the ScaledObject bounds, the unchanged six-trigger list, the static queue pool environment and the lane limits. --- packages/sie_placement/README.md | 69 ++++ .../sie_placement/src/sie_placement/helm.py | 278 +++++++++++++ .../tests/fixtures/helm/plan.json | 94 +++++ .../tests/fixtures/helm/values.json | 74 ++++ packages/sie_placement/tests/test_helm.py | 366 ++++++++++++++++++ tools/ci/tests/test_helm_render.py | 122 ++++++ 6 files changed, 1003 insertions(+) create mode 100644 packages/sie_placement/src/sie_placement/helm.py create mode 100644 packages/sie_placement/tests/fixtures/helm/plan.json create mode 100644 packages/sie_placement/tests/fixtures/helm/values.json create mode 100644 packages/sie_placement/tests/test_helm.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index cfeb962ed..25e9a7abf 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -239,6 +239,75 @@ The report renders as a `placement-report` document: - `refusals`, and `changes`: the identities and lanes that differ from `current_plan`. +## Helm values + +`helm_values` turns a plan into a values overlay for the `sie-cluster` Helm +chart, to pass with `-f` after the release's own values: + +```python +import json + +from sie_placement.helm import ChartParameters, helm_values + +with open("placement-values.json", "w", encoding="utf-8") as handle: + json.dump(helm_values(plan, ChartParameters(fallback_replicas=2)), handle, indent=2) +``` + +| Parameter | Meaning | +| --- | --- | +| `fallback_replicas` | The chart's `autoscaling.fallbackReplicas`, `2` by default | +| `bundled_dependencies` | Whether the release installs KEDA or Prometheus itself (`keda.install` or `kube-prometheus-stack.install`), which lowers the chart's lane limit from 192 to 96 | + +A plan lane names the chart lane of its `queuePool`, `machineProfile` and +`bundle`, canonicalized as the chart does: trimmed, at most 63 characters of +`[A-Za-z0-9_-]`, and lowercased. Plan lanes that canonicalize to the same three +tokens are one chart lane. The overlay sets: + +- `workers.pools.`: one enabled worker group per machine + profile, with its `machineProfile` and `queuePool`, `gpu.count` and + `gpu.product` from the cell and container shape, and `resources` requests and + limits equal to the container shape. Each chart lane is an enabled + `bundles.` entry whose `minReplicas` is the envelope's `floorMin` and + whose `maxReplicas` is `maxContainers`. +- `queueRouting.staticQueuePools.`: one static queue pool for every + queue pool other than `default`, which the gateway needs before it admits + that queue's workers. `gpus` lists every machine profile on the queue, + `minimumWorkerCount` is the lanes' `floorMin`, `pinnedModels` are their + `pinned` identities, and `bundle` names the lanes' bundle when they share + one. + +It refuses, with an `UnrealizablePlanError` that names the plan field, a plan +the chart and the gateway would not realize as planned: + +- A lane token the chart rejects, or a machine profile or bundle that is not + a lowercase DNS-1123 label, which the chart requires of worker group and + bundle keys. +- A lane with `floorMin` 0 whose `floorMax` is above 0 and below the chart's + fallback floor, `min(maxContainers, fallbackReplicas)`, which is the least + KEDA keeps on an active zero-floor lane. +- Plan lanes on one chart lane that differ in GPU product, container shape, + `maxContainers` or `floorMin`, and lanes on one machine profile that differ + in queue pool, GPU product or container shape. +- More chart lanes than the chart admits: 192 with external KEDA and + Prometheus, 96 when the release installs either. +- A `pinned` identity on the `default` queue pool, for which the gateway keeps + no static queue pool. +- Lanes on one named queue pool that differ in `floorMin` or in their pinned + identities, or that carry a floor or pins across more than one bundle. A + static queue pool applies one floor and one pinned set to every worker it + admits. +- The queue pool `_default`, which the gateway reserves. + +The chart's queue-depth threshold, cooldown and scale-down stabilization are +chart-wide, so a lane's `concurrency`, `floorMax` and buffer or scale-down +window ranges are not part of the overlay. Helm merges the overlay into the +values beneath it: a worker group or bundle that those values enable and the +plan does not place stays enabled and counts toward the lane limit, and the +worker group keys the overlay leaves out, such as `gpuType`, `nodeSelector`, +`tolerations`, `platform` and `gpu.deviceGroup`, keep their values there. +Requests reach a lane on a named queue pool through that pool, for example +`gpu="/"`. + ## Tests ```bash diff --git a/packages/sie_placement/src/sie_placement/helm.py b/packages/sie_placement/src/sie_placement/helm.py new file mode 100644 index 000000000..78d50e176 --- /dev/null +++ b/packages/sie_placement/src/sie_placement/helm.py @@ -0,0 +1,278 @@ +"""Values for the sie-cluster Helm chart that realize a placement plan.""" + +from __future__ import annotations + +import math +import re +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass, field + +from sie_placement.schema import ContainerShape, Plan, PlanLane, SchemaError, Warmth + +DEFAULT_QUEUE_POOL = "default" +RESERVED_POOL_NAME = "_default" +LANE_TOKEN_MAX_LENGTH = 63 +EXTERNAL_DEPENDENCIES_LANE_LIMIT = 192 +BUNDLED_DEPENDENCIES_LANE_LIMIT = 96 +DEFAULT_FALLBACK_REPLICAS = 2 +MILLICORES_PER_CORE = 1000 + +_LANE_TOKEN = re.compile(r"[A-Za-z0-9_-]+") +_IDENTITY_TOKEN = re.compile(r"[a-z0-9]([-a-z0-9]*[a-z0-9])?") +_GO_SPACE = ( + "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006" + "\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000" +) + + +class UnrealizablePlanError(ValueError): + def __init__(self, path: str, message: str) -> None: + super().__init__(f"{path}: {message}") + self.path = path + self.message = message + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ChartParameters: + fallback_replicas: int = DEFAULT_FALLBACK_REPLICAS + bundled_dependencies: bool = False + + def __post_init__(self) -> None: + if isinstance(self.fallback_replicas, bool) or self.fallback_replicas < 1: + raise SchemaError("fallbackReplicas", f"must be at least 1, got {self.fallback_replicas!r}") + + @property + def lane_limit(self) -> int: + return BUNDLED_DEPENDENCIES_LANE_LIMIT if self.bundled_dependencies else EXTERNAL_DEPENDENCIES_LANE_LIMIT + + @property + def dependency_topology(self) -> str: + return "bundled KEDA or Prometheus" if self.bundled_dependencies else "external KEDA and Prometheus" + + def fallback_floor(self, min_replicas: int, max_replicas: int) -> int: + return min(max_replicas, max(min_replicas, self.fallback_replicas)) + + +def lane_token(path: str, value: str) -> str: + raw = value.strip(_GO_SPACE) + if not raw: + raise UnrealizablePlanError(path, "value must not be empty or whitespace") + if len(raw.encode()) > LANE_TOKEN_MAX_LENGTH: + raise UnrealizablePlanError( + path, f"value {raw!r} exceeds the {LANE_TOKEN_MAX_LENGTH}-character physical lane limit" + ) + if _LANE_TOKEN.fullmatch(raw) is None: + raise UnrealizablePlanError(path, f"value {raw!r} must match ^[A-Za-z0-9_-]+$") + return raw.lower() + + +def _require_identity_token(path: str, role: str, value: str) -> None: + if _IDENTITY_TOKEN.fullmatch(value) is None: + raise UnrealizablePlanError(path, f"{role} {value!r} must be a lowercase Kubernetes DNS-1123 label") + + +def _cpu_quantity(path: str, cores: float) -> str: + millicores = round(cores * MILLICORES_PER_CORE) + if not math.isclose(millicores, cores * MILLICORES_PER_CORE, rel_tol=0, abs_tol=1e-6) or millicores < 1: + raise UnrealizablePlanError(path, f"must be a whole number of millicores, got {cores!r}") + if millicores % MILLICORES_PER_CORE == 0: + return str(millicores // MILLICORES_PER_CORE) + return f"{millicores}m" + + +def _resources(path: str, shape: ContainerShape) -> dict[str, str]: + return {"cpu": _cpu_quantity(f"{path}.cpuCores", shape.cpu_cores), "memory": f"{shape.memory_mib}Mi"} + + +@dataclass(slots=True, kw_only=True) +class _ChartLane: + index: int + lane: PlanLane + queue_pool: str + machine_profile: str + bundle: str + pins: dict[str, int] = field(default_factory=dict) + + @property + def path(self) -> str: + return f"$.lanes[{self.index}]" + + @property + def name(self) -> str: + return f"{self.queue_pool}/{self.machine_profile}/{self.bundle}" + + +def _pins_by_lane(plan: Plan) -> dict[str, dict[str, int]]: + pins: dict[str, dict[str, int]] = defaultdict(dict) + for index, entry in enumerate(plan.identities): + if entry.warmth is Warmth.PINNED: + pins[entry.lane][entry.identity] = index + return pins + + +def _require_realizable_floor(path: str, lane: PlanLane, parameters: ChartParameters) -> None: + envelope = lane.envelope + if envelope.floor_min or not envelope.floor_max: + return + floor = parameters.fallback_floor(0, lane.max_containers) + if envelope.floor_max < floor: + raise UnrealizablePlanError( + f"{path}.envelope.floorMax", + f"must be 0 or at least {floor}: a lane with floorMin 0 realizes a raised floor as at least the " + f"chart's fallback floor, min(maxContainers, fallbackReplicas), got {envelope.floor_max}", + ) + + +def _require_same_chart_lane(path: str, lane: PlanLane, first: _ChartLane) -> None: + other = first.lane + differences = ( + ("cell.gpuProduct", lane.cell.gpu_product, other.cell.gpu_product), + ("containerShape", lane.container_shape.to_dict(), other.container_shape.to_dict()), + ("maxContainers", lane.max_containers, other.max_containers), + ("envelope.floorMin", lane.envelope.floor_min, other.envelope.floor_min), + ) + for key, value, expected in differences: + if value != expected: + raise UnrealizablePlanError( + f"{path}.{key}", + f"names chart lane {first.name!r} with {first.path}, which has {expected!r}, got {value!r}; " + "one chart lane carries one value", + ) + + +def _chart_lanes(plan: Plan, parameters: ChartParameters) -> list[_ChartLane]: + pins = _pins_by_lane(plan) + lanes: dict[tuple[str, str, str], _ChartLane] = {} + for index, lane in enumerate(plan.lanes): + path = f"$.lanes[{index}]" + key = ( + lane_token(f"{path}.queuePool", lane.queue_pool), + lane_token(f"{path}.cell.machineProfile", lane.cell.machine_profile), + lane_token(f"{path}.bundle", lane.bundle), + ) + _require_identity_token(f"{path}.cell.machineProfile", "worker group key", key[1]) + _require_identity_token(f"{path}.bundle", "bundle key", key[2]) + _require_realizable_floor(path, lane, parameters) + chart_lane = lanes.get(key) + if chart_lane is None: + chart_lane = _ChartLane(index=index, lane=lane, queue_pool=key[0], machine_profile=key[1], bundle=key[2]) + lanes[key] = chart_lane + else: + _require_same_chart_lane(path, lane, chart_lane) + chart_lane.pins.update(pins.get(lane.name, {})) + if len(lanes) > parameters.lane_limit: + raise UnrealizablePlanError( + "$.lanes", + f"names {len(lanes)} chart lanes, above the {parameters.lane_limit} the sie-cluster chart admits " + f"with {parameters.dependency_topology}", + ) + return list(lanes.values()) + + +def _worker_group(lanes: list[_ChartLane]) -> dict[str, object]: + first = lanes[0] + for other in lanes[1:]: + differences = ( + ("queuePool", other.queue_pool, first.queue_pool), + ("cell.gpuProduct", other.lane.cell.gpu_product, first.lane.cell.gpu_product), + ("containerShape", other.lane.container_shape.to_dict(), first.lane.container_shape.to_dict()), + ) + for key, value, expected in differences: + if value != expected: + raise UnrealizablePlanError( + f"{other.path}.{key}", + f"shares worker group {first.machine_profile!r} with {first.path}, which has {expected!r}, " + f"got {value!r}; one worker group per machine profile carries one value", + ) + shape = first.lane.container_shape + return { + "enabled": True, + "machineProfile": first.machine_profile, + "queuePool": first.queue_pool, + "gpu": {"count": shape.gpu_count, "product": first.lane.cell.gpu_product}, + "resources": { + "requests": _resources(f"{first.path}.containerShape", shape), + "limits": _resources(f"{first.path}.containerShape", shape), + }, + "bundles": { + lane.bundle: { + "enabled": True, + "minReplicas": lane.lane.envelope.floor_min, + "maxReplicas": lane.lane.max_containers, + } + for lane in sorted(lanes, key=lambda lane: lane.bundle) + }, + } + + +def _static_queue_pool(lanes: list[_ChartLane]) -> dict[str, object]: + first = lanes[0] + if first.queue_pool == RESERVED_POOL_NAME: + raise UnrealizablePlanError( + f"{first.path}.queuePool", + f"{RESERVED_POOL_NAME!r} is reserved by the gateway and cannot name the lane's static queue pool", + ) + for other in lanes[1:]: + if other.lane.envelope.floor_min != first.lane.envelope.floor_min: + raise UnrealizablePlanError( + f"{other.path}.envelope.floorMin", + f"shares static queue pool {first.queue_pool!r} with {first.path}, whose floorMin is " + f"{first.lane.envelope.floor_min}, got {other.lane.envelope.floor_min}; a static queue pool " + "carries one minimumWorkerCount for every lane on it", + ) + if set(other.pins) != set(first.pins): + raise UnrealizablePlanError( + f"{other.path}.name", + f"shares static queue pool {first.queue_pool!r} with {first.path}, which pins " + f"{sorted(first.pins)!r}, but pins {sorted(other.pins)!r}; a static queue pool pins one set on " + "every lane on it", + ) + bundles = sorted({lane.bundle for lane in lanes}) + pool: dict[str, object] = {} + if len(bundles) == 1: + pool["bundle"] = bundles[0] + elif first.lane.envelope.floor_min or first.pins: + other = next(lane for lane in lanes if lane.bundle != first.bundle) + raise UnrealizablePlanError( + f"{other.path}.bundle", + f"shares static queue pool {first.queue_pool!r} with {first.path}, whose bundle is " + f"{first.bundle!r}, got {other.bundle!r}; a static queue pool carries a floor or pins for one bundle", + ) + pool["gpus"] = dict.fromkeys(sorted({lane.machine_profile for lane in lanes}), 0) + pool["minimumWorkerCount"] = first.lane.envelope.floor_min + pool["pinnedModels"] = sorted(first.pins) + return pool + + +def _require_no_default_pool_pins(lanes: Iterable[_ChartLane]) -> None: + for lane in lanes: + if lane.pins: + identity, index = min(lane.pins.items(), key=lambda item: item[1]) + raise UnrealizablePlanError( + f"$.identities[{index}].warmth", + f"pins {identity!r} on chart lane {lane.name!r}, but the gateway keeps no static queue pool for " + f"the {DEFAULT_QUEUE_POOL!r} queue pool, so its pinnedModels have no carrier; place pinned " + "identities on a named queuePool", + ) + + +def helm_values(plan: Plan, parameters: ChartParameters | None = None) -> dict[str, object]: + parameters = ChartParameters() if parameters is None else parameters + lanes = _chart_lanes(plan, parameters) + by_profile: dict[str, list[_ChartLane]] = defaultdict(list) + by_queue_pool: dict[str, list[_ChartLane]] = defaultdict(list) + for lane in lanes: + by_profile[lane.machine_profile].append(lane) + by_queue_pool[lane.queue_pool].append(lane) + pools = {profile: _worker_group(by_profile[profile]) for profile in sorted(by_profile)} + static_queue_pools: dict[str, object] = {} + for queue_pool in sorted(by_queue_pool): + if queue_pool == DEFAULT_QUEUE_POOL: + _require_no_default_pool_pins(by_queue_pool[queue_pool]) + continue + static_queue_pools[queue_pool] = _static_queue_pool(by_queue_pool[queue_pool]) + values: dict[str, object] = {"workers": {"pools": pools}} + if static_queue_pools: + values["queueRouting"] = {"staticQueuePools": static_queue_pools} + return values diff --git a/packages/sie_placement/tests/fixtures/helm/plan.json b/packages/sie_placement/tests/fixtures/helm/plan.json new file mode 100644 index 000000000..1a27cedf2 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/helm/plan.json @@ -0,0 +1,94 @@ +{ + "kind": "placement-plan", + "schemaVersion": 1, + "placement": "us-central1", + "lanes": [ + { + "name": "embed-l4", + "queuePool": "placement", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + { + "name": "generate-a100-80gb", + "queuePool": "default", + "bundle": "sglang", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 7.5, + "memoryMiB": 87040 + }, + "concurrency": 32, + "maxContainers": 3, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + }, + { + "name": "rerank-l4", + "queuePool": "Placement", + "bundle": "Default", + "cell": { + "machineProfile": "L4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 8, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 1 + } + } + ], + "identities": [ + { + "identity": "BAAI/bge-m3", + "lane": "embed-l4", + "warmth": "pinned" + }, + { + "identity": "BAAI/bge-m3:sparse", + "lane": "embed-l4", + "warmth": "lazy-resident" + }, + { + "identity": "BAAI/bge-reranker-v2-m3", + "lane": "rerank-l4", + "warmth": "pinned" + }, + { + "identity": "Qwen/Qwen3-8B", + "lane": "generate-a100-80gb", + "warmth": "cold-with-bound" + }, + { + "identity": "intfloat/e5-base-v2", + "lane": "embed-l4", + "warmth": "pinned" + } + ] +} diff --git a/packages/sie_placement/tests/fixtures/helm/values.json b/packages/sie_placement/tests/fixtures/helm/values.json new file mode 100644 index 000000000..06fe95f42 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/helm/values.json @@ -0,0 +1,74 @@ +{ + "workers": { + "pools": { + "a100-80gb": { + "enabled": true, + "machineProfile": "a100-80gb", + "queuePool": "default", + "gpu": { + "count": 1, + "product": "NVIDIA-A100-SXM4-80GB" + }, + "resources": { + "requests": { + "cpu": "7500m", + "memory": "87040Mi" + }, + "limits": { + "cpu": "7500m", + "memory": "87040Mi" + } + }, + "bundles": { + "sglang": { + "enabled": true, + "minReplicas": 0, + "maxReplicas": 3 + } + } + }, + "l4": { + "enabled": true, + "machineProfile": "l4", + "queuePool": "placement", + "gpu": { + "count": 1, + "product": "NVIDIA-L4" + }, + "resources": { + "requests": { + "cpu": "4", + "memory": "16384Mi" + }, + "limits": { + "cpu": "4", + "memory": "16384Mi" + } + }, + "bundles": { + "default": { + "enabled": true, + "minReplicas": 1, + "maxReplicas": 4 + } + } + } + } + }, + "queueRouting": { + "staticQueuePools": { + "placement": { + "bundle": "default", + "gpus": { + "l4": 0 + }, + "minimumWorkerCount": 1, + "pinnedModels": [ + "BAAI/bge-m3", + "BAAI/bge-reranker-v2-m3", + "intfloat/e5-base-v2" + ] + } + } + } +} diff --git a/packages/sie_placement/tests/test_helm.py b/packages/sie_placement/tests/test_helm.py new file mode 100644 index 000000000..582dcaa54 --- /dev/null +++ b/packages/sie_placement/tests/test_helm.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +import json +import random +from dataclasses import replace +from pathlib import Path + +import pytest +from sie_placement.helm import ( + BUNDLED_DEPENDENCIES_LANE_LIMIT, + EXTERNAL_DEPENDENCIES_LANE_LIMIT, + ChartParameters, + UnrealizablePlanError, + helm_values, + lane_token, +) +from sie_placement.report import render +from sie_placement.schema import ( + WARM_FLOOR_WARMTHS, + Cell, + ContainerShape, + Envelope, + Plan, + PlanIdentity, + PlanLane, + SchemaError, + Warmth, +) + +FIXTURES = Path(__file__).parent / "fixtures" +HELM_FIXTURES = FIXTURES / "helm" +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +WIDE = ContainerShape(gpu_count=1, cpu_cores=8, memory_mib=32768) + + +def load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def lane( + name: str, + *, + queue_pool: str = "placement", + bundle: str = "default", + cell: Cell = L4, + shape: ContainerShape = SHAPE, + max_containers: int = 4, + floor_min: int = 1, + floor_max: int | None = None, +) -> PlanLane: + return PlanLane( + name=name, + queue_pool=queue_pool, + bundle=bundle, + cell=cell, + container_shape=shape, + concurrency=8, + max_containers=max_containers, + envelope=Envelope(floor_min=floor_min, floor_max=floor_min if floor_max is None else floor_max), + ) + + +def plan_of(*lanes: PlanLane, warmths: dict[str, Warmth] | None = None) -> Plan: + warmths = warmths or {} + identities = tuple( + PlanIdentity( + identity=f"org/{entry.name}", + lane=entry.name, + warmth=warmths.get(entry.name, Warmth.PINNED if entry.envelope.floor_min else Warmth.COLD_WITH_BOUND), + ) + for entry in lanes + ) + return Plan(placement="us-central1", lanes=lanes, identities=identities) + + +def refusal(plan: Plan, parameters: ChartParameters | None = None) -> UnrealizablePlanError: + with pytest.raises(UnrealizablePlanError) as caught: + helm_values(plan, parameters) + return caught.value + + +def test_golden_plan_emits_the_golden_values() -> None: + plan = Plan.from_dict(load(HELM_FIXTURES / "plan.json")) + assert render(helm_values(plan)) == (HELM_FIXTURES / "values.json").read_text(encoding="utf-8") + + +def test_values_do_not_depend_on_document_order() -> None: + plan = Plan.from_dict(load(HELM_FIXTURES / "plan.json")) + expected = helm_values(plan) + shuffle = random.Random(5) # noqa: S311 -- reproducible test data + for _ in range(20): + lanes = list(plan.lanes) + identities = list(plan.identities) + shuffle.shuffle(lanes) + shuffle.shuffle(identities) + assert helm_values(replace(plan, lanes=tuple(lanes), identities=tuple(identities))) == expected + + +def test_lanes_naming_one_chart_lane_union_their_pins() -> None: + values = helm_values(Plan.from_dict(load(HELM_FIXTURES / "plan.json"))) + assert list(values["workers"]["pools"]["l4"]["bundles"]) == ["default"] + assert values["queueRouting"]["staticQueuePools"]["placement"]["pinnedModels"] == [ + "BAAI/bge-m3", + "BAAI/bge-reranker-v2-m3", + "intfloat/e5-base-v2", + ] + + +@pytest.mark.parametrize( + ("raw", "token"), + [ + ("l4", "l4"), + (" L4 ", "l4"), + ("\tA100_80GB\n", "a100_80gb"), + ("\u3000Default\xa0", "default"), + ("\u2028q-pool\u205f", "q-pool"), + ("a" * 63, "a" * 63), + ], +) +def test_lane_token_canonicalizes_like_the_chart(raw: str, token: str) -> None: + assert lane_token("$.x", raw) == token + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + (" \u3000", "value must not be empty or whitespace"), + ("a" * 64, "exceeds the 63-character physical lane limit"), + ("l4.x", "must match ^[A-Za-z0-9_-]+$"), + ("\x1cl4", "must match ^[A-Za-z0-9_-]+$"), + ("\u01424", "must match ^[A-Za-z0-9_-]+$"), + ("\u0142" * 32, "exceeds the 63-character physical lane limit"), + ], +) +def test_lane_token_refuses_what_the_chart_refuses(raw: str, message: str) -> None: + with pytest.raises(UnrealizablePlanError, match=r"^\$\.x: ") as caught: + lane_token("$.x", raw) + assert message in caught.value.message + + +@pytest.mark.parametrize( + ("changed", "path", "message"), + [ + ({"cell": Cell(machine_profile="l4_spot", gpu_product="NVIDIA-L4")}, "cell.machineProfile", "worker group key"), + ({"cell": Cell(machine_profile="-l4", gpu_product="NVIDIA-L4")}, "cell.machineProfile", "worker group key"), + ({"bundle": "sglang_cu130"}, "bundle", "bundle key"), + ({"bundle": "sglang-"}, "bundle", "bundle key"), + ], +) +def test_worker_group_and_bundle_keys_are_dns_labels(changed: dict, path: str, message: str) -> None: + error = refusal(plan_of(replace(lane("a"), **changed))) + assert error.path == f"$.lanes[0].{path}" + assert f"{message} " in error.message + assert "must be a lowercase Kubernetes DNS-1123 label" in error.message + + +def test_zero_floor_lane_refuses_a_raised_floor_below_the_fallback_floor() -> None: + error = refusal(plan_of(lane("a", queue_pool="default", floor_min=0, floor_max=1))) + assert error.path == "$.lanes[0].envelope.floorMax" + assert error.message.startswith("must be 0 or at least 2:") + error = refusal( + plan_of(lane("a", queue_pool="default", floor_min=0, floor_max=2)), ChartParameters(fallback_replicas=3) + ) + assert error.message.startswith("must be 0 or at least 3:") + + +@pytest.mark.parametrize( + ("floor_max", "max_containers", "fallback_replicas"), + [(0, 4, 2), (2, 4, 2), (4, 4, 2), (1, 4, 1), (1, 1, 2), (2, 2, 5)], +) +def test_zero_floor_lane_accepts_a_realizable_floor( + floor_max: int, max_containers: int, fallback_replicas: int +) -> None: + plan = plan_of(lane("a", queue_pool="default", floor_min=0, floor_max=floor_max, max_containers=max_containers)) + values = helm_values(plan, ChartParameters(fallback_replicas=fallback_replicas)) + assert values["workers"]["pools"]["l4"]["bundles"]["default"] == { + "enabled": True, + "minReplicas": 0, + "maxReplicas": max_containers, + } + + +def test_schema_fixture_plan_has_an_unrealizable_zero_floor() -> None: + error = refusal(Plan.from_dict(load(FIXTURES / "plan.json"))) + assert error.path == "$.lanes[1].envelope.floorMax" + assert error.message.startswith("must be 0 or at least 2:") + + +def test_planner_estate_plan_names_one_chart_lane_with_two_floors() -> None: + error = refusal(Plan.from_dict(load(FIXTURES / "planner" / "expected-plan.json"))) + assert error.path == "$.lanes[1].envelope.floorMin" + assert error.message.startswith("names chart lane 'default/l4/default' with $.lanes[0], which has 0, got 1") + + +def bundle_lanes(count: int, *, duplicate: bool = False) -> Plan: + lanes = [lane(f"lane-{index}", queue_pool="default", bundle=f"b{index}", floor_min=0) for index in range(count)] + if duplicate: + lanes.append(lane("lane-again", queue_pool="default", bundle="B0", floor_min=0)) + return plan_of(*lanes) + + +@pytest.mark.parametrize( + ("parameters", "limit", "topology"), + [ + (ChartParameters(), EXTERNAL_DEPENDENCIES_LANE_LIMIT, "external KEDA and Prometheus"), + (ChartParameters(bundled_dependencies=True), BUNDLED_DEPENDENCIES_LANE_LIMIT, "bundled KEDA or Prometheus"), + ], +) +def test_chart_lane_count_is_bounded_by_the_chart_limit(parameters: ChartParameters, limit: int, topology: str) -> None: + assert (EXTERNAL_DEPENDENCIES_LANE_LIMIT, BUNDLED_DEPENDENCIES_LANE_LIMIT) == (192, 96) + assert parameters.lane_limit == limit + assert len(helm_values(bundle_lanes(limit), parameters)["workers"]["pools"]["l4"]["bundles"]) == limit + assert ( + len(helm_values(bundle_lanes(limit, duplicate=True), parameters)["workers"]["pools"]["l4"]["bundles"]) == limit + ) + error = refusal(bundle_lanes(limit + 1), parameters) + assert error.path == "$.lanes" + assert error.message == ( + f"names {limit + 1} chart lanes, above the {limit} the sie-cluster chart admits with {topology}" + ) + + +def test_pins_on_the_default_queue_pool_are_refused() -> None: + plan = plan_of(lane("a", queue_pool="default"), lane("b", queue_pool="DEFAULT", bundle="sglang")) + error = refusal(plan) + assert error.path == "$.identities[0].warmth" + assert "pins 'org/a' on chart lane 'default/l4/default'" in error.message + assert "place pinned identities on a named queuePool" in error.message + + +def test_a_warm_lane_without_pins_on_the_default_queue_pool_keeps_its_floor() -> None: + plan = plan_of(lane("a", queue_pool="default", floor_min=2, floor_max=3), warmths={"a": Warmth.LAZY_RESIDENT}) + values = helm_values(plan) + assert values == { + "workers": { + "pools": { + "l4": { + "enabled": True, + "machineProfile": "l4", + "queuePool": "default", + "gpu": {"count": 1, "product": "NVIDIA-L4"}, + "resources": { + "requests": {"cpu": "4", "memory": "16384Mi"}, + "limits": {"cpu": "4", "memory": "16384Mi"}, + }, + "bundles": {"default": {"enabled": True, "minReplicas": 2, "maxReplicas": 4}}, + } + } + } + } + + +def test_a_named_queue_pool_carries_one_floor_for_every_lane_on_it() -> None: + warmths = {"a": Warmth.LAZY_RESIDENT, "b": Warmth.LAZY_RESIDENT} + plan = plan_of(lane("a"), lane("b", cell=A100, shape=WIDE), warmths=warmths) + assert helm_values(plan)["queueRouting"] == { + "staticQueuePools": { + "placement": { + "bundle": "default", + "gpus": {"a100-80gb": 0, "l4": 0}, + "minimumWorkerCount": 1, + "pinnedModels": [], + } + } + } + + +def test_a_named_queue_pool_without_floor_or_pins_admits_every_bundle() -> None: + plan = plan_of(lane("a", floor_min=0), lane("b", bundle="sglang", cell=A100, shape=WIDE, floor_min=0)) + assert helm_values(plan)["queueRouting"]["staticQueuePools"] == { + "placement": {"gpus": {"a100-80gb": 0, "l4": 0}, "minimumWorkerCount": 0, "pinnedModels": []} + } + + +@pytest.mark.parametrize( + ("second", "warmths", "path", "message"), + [ + (lane("b", cell=A100, floor_min=0), {}, "envelope.floorMin", "carries one minimumWorkerCount"), + (lane("b", cell=A100), {"b": Warmth.LAZY_RESIDENT}, "name", "pins one set on every lane"), + ( + lane("b", cell=A100, bundle="sglang"), + {"a": Warmth.LAZY_RESIDENT, "b": Warmth.LAZY_RESIDENT}, + "bundle", + "carries a floor or pins for one bundle", + ), + ], +) +def test_lanes_on_one_static_queue_pool_must_agree( + second: PlanLane, warmths: dict[str, Warmth], path: str, message: str +) -> None: + error = refusal(plan_of(lane("a"), second, warmths=warmths)) + assert error.path == f"$.lanes[1].{path}" + assert error.message.startswith("shares static queue pool 'placement' with $.lanes[0]") + assert message in error.message + + +@pytest.mark.parametrize( + ("changed", "path", "expected"), + [ + ({"cell": Cell(machine_profile="L4", gpu_product="NVIDIA-L4-24GB")}, "cell.gpuProduct", "'NVIDIA-L4'"), + ({"container_shape": WIDE}, "containerShape", "{'gpuCount': 1, 'cpuCores': 4, 'memoryMiB': 16384}"), + ({"max_containers": 5}, "maxContainers", "4"), + ({"envelope": Envelope(floor_min=2, floor_max=2)}, "envelope.floorMin", "1"), + ], +) +def test_lanes_naming_one_chart_lane_must_agree(changed: dict, path: str, expected: str) -> None: + error = refusal(plan_of(lane("a"), replace(lane("b", queue_pool="PLACEMENT", bundle="Default"), **changed))) + assert error.path == f"$.lanes[1].{path}" + assert error.message.startswith(f"names chart lane 'placement/l4/default' with $.lanes[0], which has {expected}") + + +@pytest.mark.parametrize( + ("changed", "path"), + [ + ({"queue_pool": "other"}, "queuePool"), + ({"cell": Cell(machine_profile="l4", gpu_product="NVIDIA-L4-24GB")}, "cell.gpuProduct"), + ({"container_shape": WIDE}, "containerShape"), + ], +) +def test_lanes_on_one_machine_profile_share_one_worker_group(changed: dict, path: str) -> None: + error = refusal(plan_of(lane("a", floor_min=0), replace(lane("b", bundle="sglang", floor_min=0), **changed))) + assert error.path == f"$.lanes[1].{path}" + assert error.message.startswith("shares worker group 'l4' with $.lanes[0]") + + +@pytest.mark.parametrize( + ("cores", "quantity"), + [(4, "4"), (0.5, "500m"), (7.5, "7500m"), (2.3, "2300m"), (0.001, "1m"), (12.25, "12250m")], +) +def test_cpu_cores_render_as_kubernetes_quantities(cores: float, quantity: str) -> None: + shape = ContainerShape(gpu_count=2, cpu_cores=cores, memory_mib=1536) + group = helm_values(plan_of(lane("a", queue_pool="default", floor_min=0, shape=shape)))["workers"]["pools"]["l4"] + assert group["gpu"] == {"count": 2, "product": "NVIDIA-L4"} + assert group["resources"]["requests"] == group["resources"]["limits"] == {"cpu": quantity, "memory": "1536Mi"} + + +@pytest.mark.parametrize("cores", [0.0005, 1.0001]) +def test_cpu_cores_finer_than_a_millicore_are_refused(cores: float) -> None: + shape = ContainerShape(gpu_count=1, cpu_cores=cores, memory_mib=1024) + error = refusal(plan_of(lane("a", queue_pool="default", floor_min=0, shape=shape))) + assert error.path == "$.lanes[0].containerShape.cpuCores" + + +@pytest.mark.parametrize("queue_pool", ["_default", "_DEFAULT"]) +def test_the_reserved_pool_name_cannot_carry_a_static_queue_pool(queue_pool: str) -> None: + error = refusal(plan_of(lane("a", queue_pool=queue_pool, floor_min=0))) + assert error.path == "$.lanes[0].queuePool" + assert "is reserved by the gateway" in error.message + + +@pytest.mark.parametrize("fallback_replicas", [0, -1, True]) +def test_chart_parameters_need_a_fallback_floor(fallback_replicas: int) -> None: + with pytest.raises(SchemaError, match="fallbackReplicas"): + ChartParameters(fallback_replicas=fallback_replicas) + + +def test_every_warm_floor_warmth_is_carried_by_bundle_min_replicas() -> None: + for warmth in WARM_FLOOR_WARMTHS: + plan = plan_of(lane("a", floor_min=3), warmths={"a": warmth}) + values = helm_values(plan) + assert values["workers"]["pools"]["l4"]["bundles"]["default"]["minReplicas"] == 3 + assert values["queueRouting"]["staticQueuePools"]["placement"]["minimumWorkerCount"] == 3 + pinned = ["org/a"] if warmth is Warmth.PINNED else [] + assert values["queueRouting"]["staticQueuePools"]["placement"]["pinnedModels"] == pinned diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index c7b9fd22d..b6d023cb6 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4051,3 +4051,125 @@ def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, n result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}])) assert result.returncode != 0 assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr + + +PLACEMENT_VALUES = ROOT / "packages/sie_placement/tests/fixtures/helm/values.json" +PLACEMENT_TRIGGERS = ( + "sie_gateway_pending_demand", + "sie_gateway_queue_depth", + "sie_gateway_active_lease_gpus", + "sie_gateway_pool_warm_floor", + "sie_gateway_rejected_rate", + "sie_gateway_model_loads_in_progress", +) +CHART_FALLBACK_REPLICAS = 2 +LANE_LABELS = ("pool", "machine-profile", "bundle") + + +def render_placement(tmp_path: Path, *layers: dict) -> subprocess.CompletedProcess[str]: + files = [] + for index, layer in enumerate(layers): + path = tmp_path / f"placement-{index}.yaml" + path.write_text(yaml.safe_dump(layer), encoding="utf-8") + files += ["-f", str(path)] + return subprocess.run( + ["mise", "exec", "--", "helm", "template", "sie", str(helm.CHART_DIR), "--namespace", "sie"] + + helm.validation_args(files), + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def worker_scaled_objects(docs: list[dict]) -> list[dict]: + return [ + manifest + for doc in docs + if doc.get("kind") == "ConfigMap" + for manifest in yaml.safe_load_all(doc.get("data", {}).get("scaledobjects.yaml", "")) + if manifest + and manifest["kind"] == "ScaledObject" + and manifest["spec"]["scaleTargetRef"]["kind"] == "StatefulSet" + ] + + +def test_placement_values_render_the_planned_lanes(tmp_path: Path) -> None: + overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) + base = {**AUTOSCALING_VALUES, "workers": {"pools": {"a100-80gb": {"bundles": {"default": {"enabled": False}}}}}} + result = render_placement(tmp_path, base, overlay) + assert result.returncode == 0, result.stderr + docs = [doc for doc in yaml.safe_load_all(result.stdout) if doc] + lanes = { + (group["queuePool"], group["machineProfile"], bundle): (name, bundle, settings) + for name, group in overlay["workers"]["pools"].items() + for bundle, settings in group["bundles"].items() + } + + (gpu_config,) = [doc for doc in docs if doc["metadata"]["name"] == "sie-sie-cluster-gpu-config"] + catalog = json.loads(gpu_config["data"]["configured-physical-lanes"]) + assert {(lane["pool"], lane["machineProfile"], lane["bundle"]) for lane in catalog} == set(lanes) + gateway = container_env(docs, "sie-sie-cluster-gateway", "gateway") + static_queue_pools = overlay["queueRouting"]["staticQueuePools"] + assert json.loads(gateway["SIE_GATEWAY_STATIC_QUEUE_POOLS"]["value"]) == static_queue_pools + + scaled_objects = { + tuple(scaled_object["metadata"]["labels"][f"sie.superlinked.com/{label}"] for label in LANE_LABELS): ( + scaled_object["spec"] + ) + for scaled_object in worker_scaled_objects(docs) + } + assert set(scaled_objects) == set(lanes) + for lane, (group, bundle, settings) in lanes.items(): + spec = scaled_objects[lane] + minimum, maximum = settings["minReplicas"], settings["maxReplicas"] + fallback_floor = min(maximum, max(minimum, CHART_FALLBACK_REPLICAS)) + assert spec["minReplicaCount"] == (minimum or fallback_floor) + assert spec.get("idleReplicaCount") == (None if minimum else 0) + assert spec["maxReplicaCount"] == maximum + assert spec["fallback"]["replicas"] == fallback_floor + assert [trigger["type"] for trigger in spec["triggers"]] == ["prometheus"] * len(PLACEMENT_TRIGGERS) + assert [trigger["metadata"]["metricName"] for trigger in spec["triggers"]] == [ + f"{metric}_{group}_{bundle}" for metric in PLACEMENT_TRIGGERS + ] + + pool = overlay["workers"]["pools"][group] + workload = f"sie-sie-cluster-worker-{group}-{bundle}" + (statefulset,) = [doc for doc in docs if doc["kind"] == "StatefulSet" and doc["metadata"]["name"] == workload] + (worker,) = [ + container + for container in statefulset["spec"]["template"]["spec"]["containers"] + if container["name"] == "worker" + ] + gpus = {"nvidia.com/gpu": str(pool["gpu"]["count"])} + assert worker["resources"] == {kind: pool["resources"][kind] | gpus for kind in ("requests", "limits")} + env = container_env(docs, workload, "worker") + assert (env["SIE_POOL"]["value"], env["SIE_MACHINE_PROFILE"]["value"], env["SIE_BUNDLE"]["value"]) == lane + + +def placement_lane_values(count: int) -> dict: + group = { + "enabled": True, + "machineProfile": "lanes", + "queuePool": "default", + "gpu": {"count": 1, "product": "NVIDIA-L4"}, + "resources": { + "requests": {"cpu": "4", "memory": "16384Mi"}, + "limits": {"cpu": "4", "memory": "16384Mi"}, + }, + "bundles": {f"b{index}": {"enabled": True, "minReplicas": 0, "maxReplicas": 1} for index in range(count)}, + } + return {"workers": {"pools": {"lanes": group}}} + + +@pytest.mark.parametrize( + ("dependencies", "limit", "topology"), + [({}, 192, "external KEDA and Prometheus"), ({"keda": {"install": True}}, 96, "bundled KEDA or Prometheus")], +) +def test_placement_lane_limits_match_the_chart(tmp_path: Path, dependencies: dict, limit: int, topology: str) -> None: + base = {**AUTOSCALING_VALUES, **dependencies} + accepted = render_placement(tmp_path, base, placement_lane_values(limit)) + assert accepted.returncode == 0, accepted.stderr + refused = render_placement(tmp_path, base, placement_lane_values(limit + 1)) + assert refused.returncode != 0 + assert f"configured physical lane count exceeds Helm release limit {limit} with {topology}" in refused.stderr From 8274fa2bd780a2bd259fd8994bdae28e1eb93f8a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 03:44:58 +0100 Subject: [PATCH 04/40] fix(placement): realize only the planned lanes in the Helm values The overlay merged onto worker groups that the chart's own values already define, so enabling a group also enabled its stock bundles: a release could run a lane the plan never placed, and the lane-limit check undercounted. ChartParameters now takes base_pools, the workers.pools beneath the overlay (the chart's own worker groups by default, chart_pools()). Every bundle it enables under an emitted group and the plan does not place is set to enabled: false, and the enabled lanes of the groups the overlay leaves out count toward the chart's lane limit. Two plan lanes that canonicalize to one chart lane are refused, as the chart refuses two entries on one canonical lane. Merging them realized one floor, ceiling and pinned set for lanes the plan prices separately. A container of more than one GPU is one device group or one worker per GPU, which the plan does not say. device_groups names it per machine profile, the overlay emits gpu.deviceGroup, and a multi-GPU lane on an unnamed profile is refused. The golden plan now has several bundles on one worker group, several machine profiles on one static queue pool and a device group. It renders over the chart's stock values without a masking base, and an accepted plan at the lane limit renders. The order test compares the rendered bytes. The README states that the overlay does not constrain node selection and which planner plans these rules let through. --- packages/sie_placement/README.md | 54 +++- .../sie_placement/src/sie_placement/helm.py | 164 +++++++--- .../tests/fixtures/helm/chart-pools.json | 50 ++++ .../tests/fixtures/helm/plan.json | 85 +++++- .../tests/fixtures/helm/values.json | 110 ++++++- packages/sie_placement/tests/test_helm.py | 283 +++++++++++++++--- tools/ci/tests/test_helm_render.py | 63 +++- 7 files changed, 693 insertions(+), 116 deletions(-) create mode 100644 packages/sie_placement/tests/fixtures/helm/chart-pools.json diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 25e9a7abf..4b56a6c6c 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -257,18 +257,24 @@ with open("placement-values.json", "w", encoding="utf-8") as handle: | --- | --- | | `fallback_replicas` | The chart's `autoscaling.fallbackReplicas`, `2` by default | | `bundled_dependencies` | Whether the release installs KEDA or Prometheus itself (`keda.install` or `kube-prometheus-stack.install`), which lowers the chart's lane limit from 192 to 96 | +| `base_pools` | The `workers.pools` of the values beneath the overlay: the chart's values merged with the release's own values files. Only each worker group's `enabled` and its bundles' `enabled` are read. By default the chart's own worker groups, `chart_pools()` | +| `device_groups` | Lowercase machine profile to `true` when a container of more than one GPU on it is one device group, a single worker that owns every GPU as a model too large for one accelerator needs, or `false` when it is one single-GPU worker per GPU | A plan lane names the chart lane of its `queuePool`, `machineProfile` and `bundle`, canonicalized as the chart does: trimmed, at most 63 characters of -`[A-Za-z0-9_-]`, and lowercased. Plan lanes that canonicalize to the same three -tokens are one chart lane. The overlay sets: +`[A-Za-z0-9_-]`, and lowercased. Each plan lane is a chart lane of its own. The +overlay sets: - `workers.pools.`: one enabled worker group per machine profile, with its `machineProfile` and `queuePool`, `gpu.count` and - `gpu.product` from the cell and container shape, and `resources` requests and - limits equal to the container shape. Each chart lane is an enabled - `bundles.` entry whose `minReplicas` is the envelope's `floorMin` and - whose `maxReplicas` is `maxContainers`. + `gpu.product` from the cell and container shape, `gpu.deviceGroup`, and + `resources` requests and limits equal to the container shape. Each chart + lane is an enabled `bundles.` entry whose `minReplicas` is the + envelope's `floorMin` and whose `maxReplicas` is `maxContainers`. Every other + bundle that `base_pools` enables under the group is set to + `enabled: false`, so the release runs no lane the plan does not place. + `gpu.deviceGroup` is `false` for a single GPU and the `device_groups` entry + of the machine profile for more. - `queueRouting.staticQueuePools.`: one static queue pool for every queue pool other than `default`, which the gateway needs before it admits that queue's workers. `gpus` lists every machine profile on the queue, @@ -285,11 +291,16 @@ the chart and the gateway would not realize as planned: - A lane with `floorMin` 0 whose `floorMax` is above 0 and below the chart's fallback floor, `min(maxContainers, fallbackReplicas)`, which is the least KEDA keeps on an active zero-floor lane. -- Plan lanes on one chart lane that differ in GPU product, container shape, - `maxContainers` or `floorMin`, and lanes on one machine profile that differ - in queue pool, GPU product or container shape. -- More chart lanes than the chart admits: 192 with external KEDA and - Prometheus, 96 when the release installs either. +- Two plan lanes on one chart lane. The chart refuses two entries on one + canonical lane, and a chart lane realizes one floor, one ceiling and one + pinned set, so the two lanes would share what the plan prices separately. +- Lanes on one machine profile that differ in queue pool, GPU product or + container shape. +- A container of more than one GPU on a machine profile that + `device_groups` does not name. +- More chart lanes than the chart admits, counting the lanes of the worker + groups `base_pools` enables and the overlay leaves out: 192 with external + KEDA and Prometheus, 96 when the release installs either. - A `pinned` identity on the `default` queue pool, for which the gateway keeps no static queue pool. - Lanes on one named queue pool that differ in `floorMin` or in their pinned @@ -301,13 +312,24 @@ the chart and the gateway would not realize as planned: The chart's queue-depth threshold, cooldown and scale-down stabilization are chart-wide, so a lane's `concurrency`, `floorMax` and buffer or scale-down window ranges are not part of the overlay. Helm merges the overlay into the -values beneath it: a worker group or bundle that those values enable and the -plan does not place stays enabled and counts toward the lane limit, and the -worker group keys the overlay leaves out, such as `gpuType`, `nodeSelector`, -`tolerations`, `platform` and `gpu.deviceGroup`, keep their values there. -Requests reach a lane on a named queue pool through that pool, for example +values beneath it: a worker group that those values enable and the plan does +not use stays enabled, and the worker group keys the overlay leaves out, such +as `gpuType`, `nodeSelector`, `tolerations` and `platform`, keep their values +there. No chart template reads `gpu.product`, so the overlay does not +constrain where a worker group schedules. The values beneath it must select +nodes with the cell's GPU for every machine profile the plan places, as the +cloud values files do with each worker group's `nodeSelector`. Requests reach +a lane on a named queue pool through that pool, for example `gpu="/"`. +These rules bound which plans emit. A worker group carries one queue pool, a +named queue pool carries one floor and one pinned set, and the `default` queue +pool carries no pins, so a pinned lane needs a named queue pool and a machine +profile of its own. The planner places each identity on a lane of its own, all +on the queue pool its parameters name, `default` unless set. Its plan +therefore emits only when no two identities share a machine profile and +bundle, and it pins nothing, or its queue pool is named and it has one lane. + ## Tests ```bash diff --git a/packages/sie_placement/src/sie_placement/helm.py b/packages/sie_placement/src/sie_placement/helm.py index 78d50e176..f9477ae49 100644 --- a/packages/sie_placement/src/sie_placement/helm.py +++ b/packages/sie_placement/src/sie_placement/helm.py @@ -5,7 +5,7 @@ import math import re from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from sie_placement.schema import ContainerShape, Plan, PlanLane, SchemaError, Warmth @@ -17,6 +17,14 @@ BUNDLED_DEPENDENCIES_LANE_LIMIT = 96 DEFAULT_FALLBACK_REPLICAS = 2 MILLICORES_PER_CORE = 1000 +CHART_POOL_BUNDLES = ( + ("a100-40gb", ("default",)), + ("a100-80gb", ("default",)), + ("cpu", ("default",)), + ("l4", ("default",)), + ("remote", ("remote",)), + ("sglang-cu130", ("sglang-cu130",)), +) _LANE_TOKEN = re.compile(r"[A-Za-z0-9_-]+") _IDENTITY_TOKEN = re.compile(r"[a-z0-9]([-a-z0-9]*[a-z0-9])?") @@ -33,14 +41,57 @@ def __init__(self, path: str, message: str) -> None: self.message = message +def chart_pools() -> dict[str, object]: + return { + group: {"enabled": False, "bundles": {bundle: {"enabled": True} for bundle in bundles}} + for group, bundles in CHART_POOL_BUNDLES + } + + +def _fields(path: str, value: object) -> dict[str, object]: + if not isinstance(value, Mapping): + raise SchemaError(path, f"must be a mapping, got {type(value).__name__}") + fields: dict[str, object] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise SchemaError(f"{path}[{key!r}]", "must be keyed by a string") + fields[key] = item + return fields + + +def _enabled_base_bundles(base_pools: object) -> dict[str, tuple[bool, tuple[str, ...]]]: + groups: dict[str, tuple[bool, tuple[str, ...]]] = {} + for group, value in _fields("basePools", base_pools).items(): + path = f"basePools[{group!r}]" + values = _fields(path, value) + bundles = values.get("bundles") + enabled = [ + bundle + for bundle, settings in _fields(f"{path}.bundles", {} if bundles is None else bundles).items() + if _fields(f"{path}.bundles[{bundle!r}]", {} if settings is None else settings).get("enabled", True) + ] + groups[group] = (bool(values.get("enabled")), tuple(sorted(enabled))) + return groups + + @dataclass(frozen=True, slots=True, kw_only=True) class ChartParameters: fallback_replicas: int = DEFAULT_FALLBACK_REPLICAS bundled_dependencies: bool = False + base_pools: Mapping[str, object] = field(default_factory=chart_pools) + device_groups: Mapping[str, bool] = field(default_factory=dict) def __post_init__(self) -> None: if isinstance(self.fallback_replicas, bool) or self.fallback_replicas < 1: raise SchemaError("fallbackReplicas", f"must be at least 1, got {self.fallback_replicas!r}") + _enabled_base_bundles(self.base_pools) + if not isinstance(self.device_groups, Mapping): + raise SchemaError("deviceGroups", f"must map machine profiles to true or false, got {self.device_groups!r}") + for profile, device_group in self.device_groups.items(): + if not isinstance(profile, str) or not isinstance(device_group, bool): + raise SchemaError( + "deviceGroups", f"must map a machine profile to true or false, got {profile!r}: {device_group!r}" + ) @property def lane_limit(self) -> int: @@ -124,23 +175,6 @@ def _require_realizable_floor(path: str, lane: PlanLane, parameters: ChartParame ) -def _require_same_chart_lane(path: str, lane: PlanLane, first: _ChartLane) -> None: - other = first.lane - differences = ( - ("cell.gpuProduct", lane.cell.gpu_product, other.cell.gpu_product), - ("containerShape", lane.container_shape.to_dict(), other.container_shape.to_dict()), - ("maxContainers", lane.max_containers, other.max_containers), - ("envelope.floorMin", lane.envelope.floor_min, other.envelope.floor_min), - ) - for key, value, expected in differences: - if value != expected: - raise UnrealizablePlanError( - f"{path}.{key}", - f"names chart lane {first.name!r} with {first.path}, which has {expected!r}, got {value!r}; " - "one chart lane carries one value", - ) - - def _chart_lanes(plan: Plan, parameters: ChartParameters) -> list[_ChartLane]: pins = _pins_by_lane(plan) lanes: dict[tuple[str, str, str], _ChartLane] = {} @@ -154,23 +188,59 @@ def _chart_lanes(plan: Plan, parameters: ChartParameters) -> list[_ChartLane]: _require_identity_token(f"{path}.cell.machineProfile", "worker group key", key[1]) _require_identity_token(f"{path}.bundle", "bundle key", key[2]) _require_realizable_floor(path, lane, parameters) - chart_lane = lanes.get(key) - if chart_lane is None: - chart_lane = _ChartLane(index=index, lane=lane, queue_pool=key[0], machine_profile=key[1], bundle=key[2]) - lanes[key] = chart_lane - else: - _require_same_chart_lane(path, lane, chart_lane) - chart_lane.pins.update(pins.get(lane.name, {})) - if len(lanes) > parameters.lane_limit: - raise UnrealizablePlanError( - "$.lanes", - f"names {len(lanes)} chart lanes, above the {parameters.lane_limit} the sie-cluster chart admits " - f"with {parameters.dependency_topology}", + chart_lane = _ChartLane( + index=index, + lane=lane, + queue_pool=key[0], + machine_profile=key[1], + bundle=key[2], + pins=pins.get(lane.name, {}), ) + first = lanes.setdefault(key, chart_lane) + if first is not chart_lane: + raise UnrealizablePlanError( + path, + f"names chart lane {chart_lane.name!r}, which {first.path} also names; a chart lane realizes one " + "floor, one ceiling and one pinned set, so each plan lane needs its own queuePool, machineProfile " + "and bundle", + ) return list(lanes.values()) -def _worker_group(lanes: list[_ChartLane]) -> dict[str, object]: +def _require_lane_limit( + lanes: list[_ChartLane], base: Mapping[str, tuple[bool, tuple[str, ...]]], parameters: ChartParameters +) -> None: + emitted = {lane.machine_profile for lane in lanes} + base_lanes = sum(len(bundles) for group, (enabled, bundles) in base.items() if enabled and group not in emitted) + total = len(lanes) + base_lanes + if total <= parameters.lane_limit: + return + counted = f"names {len(lanes)} chart lanes" + if base_lanes: + counted += f" and the base values enable {base_lanes} more, {total} in all" + raise UnrealizablePlanError( + "$.lanes", + f"{counted}, above the {parameters.lane_limit} the sie-cluster chart admits with " + f"{parameters.dependency_topology}", + ) + + +def _device_group(lane: _ChartLane, parameters: ChartParameters) -> bool: + count = lane.lane.container_shape.gpu_count + if count == 1: + return False + device_group = parameters.device_groups.get(lane.machine_profile) + if device_group is None: + raise UnrealizablePlanError( + f"{lane.path}.containerShape.gpuCount", + f"is {count}: the chart runs a container of {count} GPUs as one device group or as {count} " + f"single-GPU workers, and the plan does not say which; name machine profile {lane.machine_profile!r} " + "in device_groups", + ) + return device_group + + +def _worker_group(lanes: list[_ChartLane], base_bundles: tuple[str, ...], parameters: ChartParameters) -> dict: first = lanes[0] for other in lanes[1:]: differences = ( @@ -186,23 +256,30 @@ def _worker_group(lanes: list[_ChartLane]) -> dict[str, object]: f"got {value!r}; one worker group per machine profile carries one value", ) shape = first.lane.container_shape + bundles: dict[str, dict[str, object]] = { + lane.bundle: { + "enabled": True, + "minReplicas": lane.lane.envelope.floor_min, + "maxReplicas": lane.lane.max_containers, + } + for lane in lanes + } + for bundle in base_bundles: + bundles.setdefault(bundle, {"enabled": False}) return { "enabled": True, "machineProfile": first.machine_profile, "queuePool": first.queue_pool, - "gpu": {"count": shape.gpu_count, "product": first.lane.cell.gpu_product}, + "gpu": { + "count": shape.gpu_count, + "deviceGroup": _device_group(first, parameters), + "product": first.lane.cell.gpu_product, + }, "resources": { "requests": _resources(f"{first.path}.containerShape", shape), "limits": _resources(f"{first.path}.containerShape", shape), }, - "bundles": { - lane.bundle: { - "enabled": True, - "minReplicas": lane.lane.envelope.floor_min, - "maxReplicas": lane.lane.max_containers, - } - for lane in sorted(lanes, key=lambda lane: lane.bundle) - }, + "bundles": dict(sorted(bundles.items())), } @@ -260,12 +337,17 @@ def _require_no_default_pool_pins(lanes: Iterable[_ChartLane]) -> None: def helm_values(plan: Plan, parameters: ChartParameters | None = None) -> dict[str, object]: parameters = ChartParameters() if parameters is None else parameters lanes = _chart_lanes(plan, parameters) + base = _enabled_base_bundles(parameters.base_pools) + _require_lane_limit(lanes, base, parameters) by_profile: dict[str, list[_ChartLane]] = defaultdict(list) by_queue_pool: dict[str, list[_ChartLane]] = defaultdict(list) for lane in lanes: by_profile[lane.machine_profile].append(lane) by_queue_pool[lane.queue_pool].append(lane) - pools = {profile: _worker_group(by_profile[profile]) for profile in sorted(by_profile)} + pools = { + profile: _worker_group(by_profile[profile], base.get(profile, (False, ()))[1], parameters) + for profile in sorted(by_profile) + } static_queue_pools: dict[str, object] = {} for queue_pool in sorted(by_queue_pool): if queue_pool == DEFAULT_QUEUE_POOL: diff --git a/packages/sie_placement/tests/fixtures/helm/chart-pools.json b/packages/sie_placement/tests/fixtures/helm/chart-pools.json new file mode 100644 index 000000000..a72b0bfea --- /dev/null +++ b/packages/sie_placement/tests/fixtures/helm/chart-pools.json @@ -0,0 +1,50 @@ +{ + "a100-40gb": { + "enabled": false, + "bundles": { + "default": { + "enabled": true + } + } + }, + "a100-80gb": { + "enabled": false, + "bundles": { + "default": { + "enabled": true + } + } + }, + "cpu": { + "enabled": false, + "bundles": { + "default": { + "enabled": true + } + } + }, + "l4": { + "enabled": false, + "bundles": { + "default": { + "enabled": true + } + } + }, + "remote": { + "enabled": false, + "bundles": { + "remote": { + "enabled": true + } + } + }, + "sglang-cu130": { + "enabled": false, + "bundles": { + "sglang-cu130": { + "enabled": true + } + } + } +} diff --git a/packages/sie_placement/tests/fixtures/helm/plan.json b/packages/sie_placement/tests/fixtures/helm/plan.json index 1a27cedf2..342cff889 100644 --- a/packages/sie_placement/tests/fixtures/helm/plan.json +++ b/packages/sie_placement/tests/fixtures/helm/plan.json @@ -44,11 +44,11 @@ } }, { - "name": "rerank-l4", - "queuePool": "Placement", + "name": "rerank-l4-spot", + "queuePool": "Rerank", "bundle": "Default", "cell": { - "machineProfile": "L4", + "machineProfile": "L4-Spot", "gpuProduct": "NVIDIA-L4" }, "containerShape": { @@ -57,11 +57,71 @@ "memoryMiB": 16384 }, "concurrency": 8, - "maxContainers": 4, + "maxContainers": 2, "envelope": { "floorMin": 1, "floorMax": 1 } + }, + { + "name": "embed-a100-80gb", + "queuePool": "Default", + "bundle": "default", + "cell": { + "machineProfile": "a100-80gb", + "gpuProduct": "NVIDIA-A100-SXM4-80GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 7.5, + "memoryMiB": 87040 + }, + "concurrency": 16, + "maxContainers": 2, + "envelope": { + "floorMin": 0, + "floorMax": 0 + } + }, + { + "name": "batch-h100", + "queuePool": "Batch", + "bundle": "default", + "cell": { + "machineProfile": "H100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3" + }, + "containerShape": { + "gpuCount": 2, + "cpuCores": 16, + "memoryMiB": 196608 + }, + "concurrency": 32, + "maxContainers": 2, + "envelope": { + "floorMin": 0, + "floorMax": 2 + } + }, + { + "name": "batch-a100-40gb", + "queuePool": "batch", + "bundle": "sglang", + "cell": { + "machineProfile": "a100-40gb", + "gpuProduct": "NVIDIA-A100-SXM4-40GB" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 8, + "memoryMiB": 65536 + }, + "concurrency": 16, + "maxContainers": 2, + "envelope": { + "floorMin": 0, + "floorMax": 0 + } } ], "identities": [ @@ -77,14 +137,29 @@ }, { "identity": "BAAI/bge-reranker-v2-m3", - "lane": "rerank-l4", + "lane": "rerank-l4-spot", "warmth": "pinned" }, + { + "identity": "Qwen/Qwen3-32B", + "lane": "batch-h100", + "warmth": "cold-with-bound" + }, + { + "identity": "Qwen/Qwen3-4B", + "lane": "batch-a100-40gb", + "warmth": "cold-with-bound" + }, { "identity": "Qwen/Qwen3-8B", "lane": "generate-a100-80gb", "warmth": "cold-with-bound" }, + { + "identity": "Qwen/Qwen3-Embedding-4B", + "lane": "embed-a100-80gb", + "warmth": "cold-with-bound" + }, { "identity": "intfloat/e5-base-v2", "lane": "embed-l4", diff --git a/packages/sie_placement/tests/fixtures/helm/values.json b/packages/sie_placement/tests/fixtures/helm/values.json index 06fe95f42..fd745d118 100644 --- a/packages/sie_placement/tests/fixtures/helm/values.json +++ b/packages/sie_placement/tests/fixtures/helm/values.json @@ -1,12 +1,43 @@ { "workers": { "pools": { + "a100-40gb": { + "enabled": true, + "machineProfile": "a100-40gb", + "queuePool": "batch", + "gpu": { + "count": 1, + "deviceGroup": false, + "product": "NVIDIA-A100-SXM4-40GB" + }, + "resources": { + "requests": { + "cpu": "8", + "memory": "65536Mi" + }, + "limits": { + "cpu": "8", + "memory": "65536Mi" + } + }, + "bundles": { + "default": { + "enabled": false + }, + "sglang": { + "enabled": true, + "minReplicas": 0, + "maxReplicas": 2 + } + } + }, "a100-80gb": { "enabled": true, "machineProfile": "a100-80gb", "queuePool": "default", "gpu": { "count": 1, + "deviceGroup": false, "product": "NVIDIA-A100-SXM4-80GB" }, "resources": { @@ -20,6 +51,11 @@ } }, "bundles": { + "default": { + "enabled": true, + "minReplicas": 0, + "maxReplicas": 2 + }, "sglang": { "enabled": true, "minReplicas": 0, @@ -27,12 +63,40 @@ } } }, + "h100": { + "enabled": true, + "machineProfile": "h100", + "queuePool": "batch", + "gpu": { + "count": 2, + "deviceGroup": true, + "product": "NVIDIA-H100-80GB-HBM3" + }, + "resources": { + "requests": { + "cpu": "16", + "memory": "196608Mi" + }, + "limits": { + "cpu": "16", + "memory": "196608Mi" + } + }, + "bundles": { + "default": { + "enabled": true, + "minReplicas": 0, + "maxReplicas": 2 + } + } + }, "l4": { "enabled": true, "machineProfile": "l4", "queuePool": "placement", "gpu": { "count": 1, + "deviceGroup": false, "product": "NVIDIA-L4" }, "resources": { @@ -52,11 +116,46 @@ "maxReplicas": 4 } } + }, + "l4-spot": { + "enabled": true, + "machineProfile": "l4-spot", + "queuePool": "rerank", + "gpu": { + "count": 1, + "deviceGroup": false, + "product": "NVIDIA-L4" + }, + "resources": { + "requests": { + "cpu": "4", + "memory": "16384Mi" + }, + "limits": { + "cpu": "4", + "memory": "16384Mi" + } + }, + "bundles": { + "default": { + "enabled": true, + "minReplicas": 1, + "maxReplicas": 2 + } + } } } }, "queueRouting": { "staticQueuePools": { + "batch": { + "gpus": { + "a100-40gb": 0, + "h100": 0 + }, + "minimumWorkerCount": 0, + "pinnedModels": [] + }, "placement": { "bundle": "default", "gpus": { @@ -65,9 +164,18 @@ "minimumWorkerCount": 1, "pinnedModels": [ "BAAI/bge-m3", - "BAAI/bge-reranker-v2-m3", "intfloat/e5-base-v2" ] + }, + "rerank": { + "bundle": "default", + "gpus": { + "l4-spot": 0 + }, + "minimumWorkerCount": 1, + "pinnedModels": [ + "BAAI/bge-reranker-v2-m3" + ] } } } diff --git a/packages/sie_placement/tests/test_helm.py b/packages/sie_placement/tests/test_helm.py index 582dcaa54..b0ea6dd32 100644 --- a/packages/sie_placement/tests/test_helm.py +++ b/packages/sie_placement/tests/test_helm.py @@ -4,26 +4,34 @@ import random from dataclasses import replace from pathlib import Path +from typing import Any import pytest +from sie_placement import planner from sie_placement.helm import ( BUNDLED_DEPENDENCIES_LANE_LIMIT, EXTERNAL_DEPENDENCIES_LANE_LIMIT, ChartParameters, UnrealizablePlanError, + chart_pools, helm_values, lane_token, ) from sie_placement.report import render from sie_placement.schema import ( WARM_FLOOR_WARMTHS, + BootEvidence, Cell, ContainerShape, + Demand, Envelope, Plan, PlanIdentity, PlanLane, + PriceList, + Receipt, SchemaError, + ServiceLevels, Warmth, ) @@ -33,6 +41,16 @@ A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) WIDE = ContainerShape(gpu_count=1, cpu_cores=8, memory_mib=32768) +PAIR = ContainerShape(gpu_count=2, cpu_cores=16, memory_mib=65536) +GOLDEN_PARAMETERS = ChartParameters(device_groups={"h100": True}) +PLANNER_PARAMETERS = planner.PlannerParameters( + placement="us-central1", + max_containers=4, + fallback_floor=2, + cooldown_s=900, + buffer_containers=1, + bundles={"Qwen/Qwen3-8B": "sglang"}, +) def load(path: Path) -> dict: @@ -81,31 +99,79 @@ def refusal(plan: Plan, parameters: ChartParameters | None = None) -> Unrealizab return caught.value +def golden_plan() -> Plan: + return Plan.from_dict(load(HELM_FIXTURES / "plan.json")) + + +def golden_values() -> dict: + return load(HELM_FIXTURES / "values.json") + + def test_golden_plan_emits_the_golden_values() -> None: - plan = Plan.from_dict(load(HELM_FIXTURES / "plan.json")) - assert render(helm_values(plan)) == (HELM_FIXTURES / "values.json").read_text(encoding="utf-8") + assert render(helm_values(golden_plan(), GOLDEN_PARAMETERS)) == (HELM_FIXTURES / "values.json").read_text( + encoding="utf-8" + ) def test_values_do_not_depend_on_document_order() -> None: - plan = Plan.from_dict(load(HELM_FIXTURES / "plan.json")) - expected = helm_values(plan) + plan = golden_plan() + expected = (HELM_FIXTURES / "values.json").read_text(encoding="utf-8") shuffle = random.Random(5) # noqa: S311 -- reproducible test data for _ in range(20): lanes = list(plan.lanes) identities = list(plan.identities) shuffle.shuffle(lanes) shuffle.shuffle(identities) - assert helm_values(replace(plan, lanes=tuple(lanes), identities=tuple(identities))) == expected + shuffled = replace(plan, lanes=tuple(lanes), identities=tuple(identities)) + assert render(helm_values(shuffled, GOLDEN_PARAMETERS)) == expected + + +def test_the_default_base_is_the_charts_own_worker_groups() -> None: + assert ChartParameters().base_pools == chart_pools() == load(HELM_FIXTURES / "chart-pools.json") + assert chart_pools() is not chart_pools() -def test_lanes_naming_one_chart_lane_union_their_pins() -> None: - values = helm_values(Plan.from_dict(load(HELM_FIXTURES / "plan.json"))) - assert list(values["workers"]["pools"]["l4"]["bundles"]) == ["default"] - assert values["queueRouting"]["staticQueuePools"]["placement"]["pinnedModels"] == [ - "BAAI/bge-m3", - "BAAI/bge-reranker-v2-m3", - "intfloat/e5-base-v2", - ] +def test_an_emitted_worker_group_disables_the_base_bundles_the_plan_does_not_place() -> None: + bundles = golden_values()["workers"]["pools"]["a100-40gb"]["bundles"] + assert bundles == {"default": {"enabled": False}, "sglang": {"enabled": True, "minReplicas": 0, "maxReplicas": 2}} + base = chart_pools() | { + "l4": {"enabled": False, "bundles": {"default": {}, "sglang": None, "old": {"enabled": False}}}, + } + values = helm_values(plan_of(lane("a", bundle="candle")), ChartParameters(base_pools=base)) + assert values["workers"]["pools"]["l4"]["bundles"] == { + "candle": {"enabled": True, "minReplicas": 1, "maxReplicas": 4}, + "default": {"enabled": False}, + "sglang": {"enabled": False}, + } + + +@pytest.mark.parametrize( + ("candidate", "path", "message"), + [ + ( + plan_of( + lane("a"), lane("b", queue_pool="PLACEMENT", bundle="Default", cell=replace(L4, machine_profile="L4")) + ), + "$.lanes[1]", + "names chart lane 'placement/l4/default', which $.lanes[0] also names;", + ), + ( + plan_of(lane("a", queue_pool="default", floor_min=0), lane("b", queue_pool="default", floor_min=0)), + "$.lanes[1]", + "names chart lane 'default/l4/default', which $.lanes[0] also names;", + ), + ( + plan_of(lane("a", bundle="sglang"), lane("b"), lane("c", bundle="SGLang")), + "$.lanes[2]", + "names chart lane 'placement/l4/sglang', which $.lanes[0] also names;", + ), + ], +) +def test_two_plan_lanes_on_one_chart_lane_are_refused(candidate: Plan, path: str, message: str) -> None: + error = refusal(candidate) + assert error.path == path + assert error.message.startswith(message) + assert error.message.endswith("so each plan lane needs its own queuePool, machineProfile and bundle") @pytest.mark.parametrize( @@ -188,17 +254,69 @@ def test_schema_fixture_plan_has_an_unrealizable_zero_floor() -> None: assert error.message.startswith("must be 0 or at least 2:") -def test_planner_estate_plan_names_one_chart_lane_with_two_floors() -> None: +def test_planner_estate_plan_names_one_chart_lane_twice() -> None: error = refusal(Plan.from_dict(load(FIXTURES / "planner" / "expected-plan.json"))) - assert error.path == "$.lanes[1].envelope.floorMin" - assert error.message.startswith("names chart lane 'default/l4/default' with $.lanes[0], which has 0, got 1") + assert error.path == "$.lanes[1]" + assert error.message.startswith("names chart lane 'default/l4/default', which $.lanes[0] also names;") + + +def planner_estate(identities: set[str], parameters: planner.PlannerParameters) -> planner.PlannerResult: + service_levels = ServiceLevels.from_dict(load(FIXTURES / "planner" / "service-levels.json")) + receipts = [Receipt.from_dict(document) for document in load(FIXTURES / "planner" / "receipts.json")] + boots = [BootEvidence.from_dict(document) for document in load(FIXTURES / "planner" / "boot-evidence.json")] + return planner.plan( + replace(service_levels, identities=tuple(e for e in service_levels.identities if e.identity in identities)), + [entry for entry in receipts if entry.identity in identities], + [entry for entry in boots if entry.identity in identities], + PriceList.from_dict(load(FIXTURES / "planner" / "price-list.json")), + Demand.from_dict(load(FIXTURES / "planner" / "demand.json")), + None, + parameters, + ) + + +def test_a_planner_plan_with_one_lane_per_chart_lane_emits_its_lanes() -> None: + result = planner_estate({"BAAI/bge-m3:sparse", "Qwen/Qwen3-8B"}, PLANNER_PARAMETERS) + assert {entry.warmth for entry in result.plan.identities} == {Warmth.COLD_WITH_BOUND} + values = helm_values(result.plan, ChartParameters(fallback_replicas=PLANNER_PARAMETERS.fallback_floor)) + emitted = { + (group["queuePool"], profile, bundle): (settings["minReplicas"], settings["maxReplicas"]) + for profile, group in values["workers"]["pools"].items() + for bundle, settings in group["bundles"].items() + if settings["enabled"] + } + assert emitted == { + (entry.queue_pool, entry.cell.machine_profile, entry.bundle): (entry.envelope.floor_min, entry.max_containers) + for entry in result.plan.lanes + } + assert len(emitted) == 2 + assert "queueRouting" not in values + +@pytest.mark.parametrize( + ("queue_pool", "path", "message"), + [ + ("default", "$.identities[0].warmth", "pins 'BAAI/bge-m3' on chart lane 'default/l4/default', but the"), + ("placement", "$.lanes[1].envelope.floorMin", "shares static queue pool 'placement' with $.lanes[0], whose"), + ], +) +def test_a_planner_plan_that_pins_beside_another_lane_is_refused(queue_pool: str, path: str, message: str) -> None: + result = planner_estate({"BAAI/bge-m3", "Qwen/Qwen3-8B"}, replace(PLANNER_PARAMETERS, queue_pool=queue_pool)) + assert [entry.warmth for entry in result.plan.identities] == [Warmth.PINNED, Warmth.COLD_WITH_BOUND] + error = refusal(result.plan) + assert error.path == path + assert error.message.startswith(message) -def bundle_lanes(count: int, *, duplicate: bool = False) -> Plan: - lanes = [lane(f"lane-{index}", queue_pool="default", bundle=f"b{index}", floor_min=0) for index in range(count)] - if duplicate: - lanes.append(lane("lane-again", queue_pool="default", bundle="B0", floor_min=0)) - return plan_of(*lanes) + +def bundle_lanes(count: int) -> Plan: + return plan_of( + *(lane(f"lane-{index}", queue_pool="default", bundle=f"b{index}", floor_min=0) for index in range(count)) + ) + + +def base_lanes(count: int, *, enabled: bool = True, more: dict | None = None) -> dict[str, object]: + bundles: dict[str, dict] = {f"b{index}": {} for index in range(count)} + return chart_pools() | {"lanes": {"enabled": enabled, "bundles": bundles | (more or {})}} @pytest.mark.parametrize( @@ -211,10 +329,10 @@ def bundle_lanes(count: int, *, duplicate: bool = False) -> Plan: def test_chart_lane_count_is_bounded_by_the_chart_limit(parameters: ChartParameters, limit: int, topology: str) -> None: assert (EXTERNAL_DEPENDENCIES_LANE_LIMIT, BUNDLED_DEPENDENCIES_LANE_LIMIT) == (192, 96) assert parameters.lane_limit == limit - assert len(helm_values(bundle_lanes(limit), parameters)["workers"]["pools"]["l4"]["bundles"]) == limit - assert ( - len(helm_values(bundle_lanes(limit, duplicate=True), parameters)["workers"]["pools"]["l4"]["bundles"]) == limit - ) + bundles = helm_values(bundle_lanes(limit), parameters)["workers"]["pools"]["l4"]["bundles"] + assert bundles.pop("default") == {"enabled": False} + assert len(bundles) == limit + assert all(settings["enabled"] for settings in bundles.values()) error = refusal(bundle_lanes(limit + 1), parameters) assert error.path == "$.lanes" assert error.message == ( @@ -222,6 +340,68 @@ def test_chart_lane_count_is_bounded_by_the_chart_limit(parameters: ChartParamet ) +@pytest.mark.parametrize( + ("bundled_dependencies", "limit", "topology"), + [ + (False, EXTERNAL_DEPENDENCIES_LANE_LIMIT, "external KEDA and Prometheus"), + (True, BUNDLED_DEPENDENCIES_LANE_LIMIT, "bundled KEDA or Prometheus"), + ], +) +def test_lanes_the_base_values_enable_count_toward_the_chart_limit( + bundled_dependencies: bool, limit: int, topology: str +) -> None: + planned = sum( + settings["enabled"] + for group in golden_values()["workers"]["pools"].values() + for settings in group["bundles"].values() + ) + assert planned == len(golden_plan().lanes) + accepted = replace( + GOLDEN_PARAMETERS, bundled_dependencies=bundled_dependencies, base_pools=base_lanes(limit - planned) + ) + assert helm_values(golden_plan(), accepted) == golden_values() + refused = replace(accepted, base_pools=base_lanes(limit - planned + 1)) + error = refusal(golden_plan(), refused) + assert error.path == "$.lanes" + assert error.message == ( + f"names {planned} chart lanes and the base values enable {limit - planned + 1} more, {limit + 1} in all, " + f"above the {limit} the sie-cluster chart admits with {topology}" + ) + + +def test_only_enabled_lanes_of_worker_groups_the_overlay_leaves_out_count_toward_the_chart_limit() -> None: + limit = EXTERNAL_DEPENDENCIES_LANE_LIMIT + disabled = replace(GOLDEN_PARAMETERS, base_pools=base_lanes(limit, enabled=False)) + assert helm_values(golden_plan(), disabled) == golden_values() + l4 = {"l4": {"enabled": True, "bundles": {"default": {}, "extra": {}}}} + one = plan_of(lane("a", queue_pool="default", floor_min=0)) + base = base_lanes(limit - 1, more={"off": {"enabled": False}}) | l4 + assert helm_values(one, ChartParameters(base_pools=base))["workers"]["pools"]["l4"]["bundles"] == { + "default": {"enabled": True, "minReplicas": 0, "maxReplicas": 4}, + "extra": {"enabled": False}, + } + base = base_lanes(limit - 1, more={"off": {"enabled": False}, "on": {"enabled": True}}) | l4 + assert refusal(one, ChartParameters(base_pools=base)).message.startswith( + f"names 1 chart lanes and the base values enable {limit} more, {limit + 1} in all" + ) + + +@pytest.mark.parametrize( + ("base_pools", "path"), + [ + ([], "basePools"), + ({"l4": []}, "basePools['l4']"), + ({1: {}}, "basePools[1]"), + ({"l4": {"bundles": []}}, "basePools['l4'].bundles"), + ({"l4": {"bundles": {"default": []}}}, "basePools['l4'].bundles['default']"), + ], +) +def test_base_pools_must_be_worker_group_values(base_pools: Any, path: str) -> None: + with pytest.raises(SchemaError) as caught: + ChartParameters(base_pools=base_pools) + assert caught.value.path == path + + def test_pins_on_the_default_queue_pool_are_refused() -> None: plan = plan_of(lane("a", queue_pool="default"), lane("b", queue_pool="DEFAULT", bundle="sglang")) error = refusal(plan) @@ -240,7 +420,7 @@ def test_a_warm_lane_without_pins_on_the_default_queue_pool_keeps_its_floor() -> "enabled": True, "machineProfile": "l4", "queuePool": "default", - "gpu": {"count": 1, "product": "NVIDIA-L4"}, + "gpu": {"count": 1, "deviceGroup": False, "product": "NVIDIA-L4"}, "resources": { "requests": {"cpu": "4", "memory": "16384Mi"}, "limits": {"cpu": "4", "memory": "16384Mi"}, @@ -296,19 +476,10 @@ def test_lanes_on_one_static_queue_pool_must_agree( assert message in error.message -@pytest.mark.parametrize( - ("changed", "path", "expected"), - [ - ({"cell": Cell(machine_profile="L4", gpu_product="NVIDIA-L4-24GB")}, "cell.gpuProduct", "'NVIDIA-L4'"), - ({"container_shape": WIDE}, "containerShape", "{'gpuCount': 1, 'cpuCores': 4, 'memoryMiB': 16384}"), - ({"max_containers": 5}, "maxContainers", "4"), - ({"envelope": Envelope(floor_min=2, floor_max=2)}, "envelope.floorMin", "1"), - ], -) -def test_lanes_naming_one_chart_lane_must_agree(changed: dict, path: str, expected: str) -> None: - error = refusal(plan_of(lane("a"), replace(lane("b", queue_pool="PLACEMENT", bundle="Default"), **changed))) - assert error.path == f"$.lanes[1].{path}" - assert error.message.startswith(f"names chart lane 'placement/l4/default' with $.lanes[0], which has {expected}") +def test_two_pinned_lanes_need_two_named_queue_pools() -> None: + error = refusal(plan_of(lane("a"), lane("b", cell=A100, shape=WIDE))) + assert error.path == "$.lanes[1].name" + assert "which pins ['org/a'], but pins ['org/b']" in error.message @pytest.mark.parametrize( @@ -330,12 +501,42 @@ def test_lanes_on_one_machine_profile_share_one_worker_group(changed: dict, path [(4, "4"), (0.5, "500m"), (7.5, "7500m"), (2.3, "2300m"), (0.001, "1m"), (12.25, "12250m")], ) def test_cpu_cores_render_as_kubernetes_quantities(cores: float, quantity: str) -> None: - shape = ContainerShape(gpu_count=2, cpu_cores=cores, memory_mib=1536) + shape = ContainerShape(gpu_count=1, cpu_cores=cores, memory_mib=1536) group = helm_values(plan_of(lane("a", queue_pool="default", floor_min=0, shape=shape)))["workers"]["pools"]["l4"] - assert group["gpu"] == {"count": 2, "product": "NVIDIA-L4"} + assert group["gpu"] == {"count": 1, "deviceGroup": False, "product": "NVIDIA-L4"} assert group["resources"]["requests"] == group["resources"]["limits"] == {"cpu": quantity, "memory": "1536Mi"} +@pytest.mark.parametrize("device_group", [True, False]) +def test_a_multi_gpu_container_is_the_device_group_the_parameters_name(device_group: bool) -> None: + plan = plan_of(lane("a", queue_pool="default", floor_min=0, shape=PAIR), lane("b", cell=A100, shape=PAIR)) + values = helm_values(plan, ChartParameters(device_groups={"l4": device_group, "a100-80gb": not device_group})) + pools = values["workers"]["pools"] + assert pools["l4"]["gpu"] == {"count": 2, "deviceGroup": device_group, "product": "NVIDIA-L4"} + assert pools["a100-80gb"]["gpu"] == {"count": 2, "deviceGroup": not device_group, "product": A100.gpu_product} + + +def test_a_multi_gpu_container_needs_its_device_group_named() -> None: + plan = plan_of(lane("a", queue_pool="default", floor_min=0), lane("b", cell=A100, shape=PAIR)) + error = refusal(plan, ChartParameters(device_groups={"l4": True, "A100-80GB": True})) + assert error.path == "$.lanes[1].containerShape.gpuCount" + assert error.message == ( + "is 2: the chart runs a container of 2 GPUs as one device group or as 2 single-GPU workers, and the plan " + "does not say which; name machine profile 'a100-80gb' in device_groups" + ) + + +def test_a_single_gpu_container_is_never_a_device_group() -> None: + values = helm_values(plan_of(lane("a")), ChartParameters(device_groups={"l4": True})) + assert values["workers"]["pools"]["l4"]["gpu"]["deviceGroup"] is False + + +@pytest.mark.parametrize("device_groups", [{"l4": 1}, {1: True}, [("l4", True)]]) +def test_device_groups_map_machine_profiles_to_booleans(device_groups: Any) -> None: + with pytest.raises(SchemaError, match="deviceGroups"): + ChartParameters(device_groups=device_groups) + + @pytest.mark.parametrize("cores", [0.0005, 1.0001]) def test_cpu_cores_finer_than_a_millicore_are_refused(cores: float) -> None: shape = ContainerShape(gpu_count=1, cpu_cores=cores, memory_mib=1024) diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index b6d023cb6..15db7f278 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4053,7 +4053,8 @@ def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, n assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr -PLACEMENT_VALUES = ROOT / "packages/sie_placement/tests/fixtures/helm/values.json" +PLACEMENT_FIXTURES = ROOT / "packages/sie_placement/tests/fixtures/helm" +PLACEMENT_VALUES = PLACEMENT_FIXTURES / "values.json" PLACEMENT_TRIGGERS = ( "sie_gateway_pending_demand", "sie_gateway_queue_depth", @@ -4094,21 +4095,50 @@ def worker_scaled_objects(docs: list[dict]) -> list[dict]: ] -def test_placement_values_render_the_planned_lanes(tmp_path: Path) -> None: - overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) - base = {**AUTOSCALING_VALUES, "workers": {"pools": {"a100-80gb": {"bundles": {"default": {"enabled": False}}}}}} - result = render_placement(tmp_path, base, overlay) - assert result.returncode == 0, result.stderr - docs = [doc for doc in yaml.safe_load_all(result.stdout) if doc] - lanes = { +def placement_lanes(overlay: dict) -> dict[tuple[str, str, str], tuple[str, str, dict]]: + return { (group["queuePool"], group["machineProfile"], bundle): (name, bundle, settings) for name, group in overlay["workers"]["pools"].items() for bundle, settings in group["bundles"].items() + if settings["enabled"] } + +def configured_lanes(docs: list[dict]) -> list[tuple[str, str, str]]: (gpu_config,) = [doc for doc in docs if doc["metadata"]["name"] == "sie-sie-cluster-gpu-config"] catalog = json.loads(gpu_config["data"]["configured-physical-lanes"]) - assert {(lane["pool"], lane["machineProfile"], lane["bundle"]) for lane in catalog} == set(lanes) + return [(lane["pool"], lane["machineProfile"], lane["bundle"]) for lane in catalog] + + +def test_the_placement_chart_pools_are_the_charts_worker_groups() -> None: + values = yaml.safe_load((helm.CHART_DIR / "values.yaml").read_text(encoding="utf-8")) + pools = { + name: { + "enabled": bool(group.get("enabled")), + "bundles": { + bundle: {"enabled": bool((settings or {}).get("enabled", True))} + for bundle, settings in group["bundles"].items() + }, + } + for name, group in values["workers"]["pools"].items() + } + assert pools == json.loads((PLACEMENT_FIXTURES / "chart-pools.json").read_text(encoding="utf-8")) + + +def test_placement_values_render_the_planned_lanes(tmp_path: Path) -> None: + overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) + result = render_placement(tmp_path, AUTOSCALING_VALUES, overlay) + assert result.returncode == 0, result.stderr + docs = [doc for doc in yaml.safe_load_all(result.stdout) if doc] + lanes = placement_lanes(overlay) + assert any( + not settings["enabled"] + for group in overlay["workers"]["pools"].values() + for settings in group["bundles"].values() + ) + assert any(group["gpu"]["deviceGroup"] for group in overlay["workers"]["pools"].values()) + + assert sorted(configured_lanes(docs)) == sorted(lanes) gateway = container_env(docs, "sie-sie-cluster-gateway", "gateway") static_queue_pools = overlay["queueRouting"]["staticQueuePools"] assert json.loads(gateway["SIE_GATEWAY_STATIC_QUEUE_POOLS"]["value"]) == static_queue_pools @@ -4143,6 +4173,10 @@ def test_placement_values_render_the_planned_lanes(tmp_path: Path) -> None: ] gpus = {"nvidia.com/gpu": str(pool["gpu"]["count"])} assert worker["resources"] == {kind: pool["resources"][kind] | gpus for kind in ("requests", "limits")} + assert [container["name"] for container in statefulset["spec"]["template"]["spec"]["containers"]] == [ + "worker", + "worker-sidecar", + ] env = container_env(docs, workload, "worker") assert (env["SIE_POOL"]["value"], env["SIE_MACHINE_PROFILE"]["value"], env["SIE_BUNDLE"]["value"]) == lane @@ -4166,10 +4200,15 @@ def placement_lane_values(count: int) -> dict: ("dependencies", "limit", "topology"), [({}, 192, "external KEDA and Prometheus"), ({"keda": {"install": True}}, 96, "bundled KEDA or Prometheus")], ) -def test_placement_lane_limits_match_the_chart(tmp_path: Path, dependencies: dict, limit: int, topology: str) -> None: +def test_placement_values_at_the_lane_limit_render( + tmp_path: Path, dependencies: dict, limit: int, topology: str +) -> None: + overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) + planned = len(placement_lanes(overlay)) base = {**AUTOSCALING_VALUES, **dependencies} - accepted = render_placement(tmp_path, base, placement_lane_values(limit)) + accepted = render_placement(tmp_path, base, placement_lane_values(limit - planned), overlay) assert accepted.returncode == 0, accepted.stderr - refused = render_placement(tmp_path, base, placement_lane_values(limit + 1)) + assert len(configured_lanes([doc for doc in yaml.safe_load_all(accepted.stdout) if doc])) == limit + refused = render_placement(tmp_path, base, placement_lane_values(limit - planned + 1), overlay) assert refused.returncode != 0 assert f"configured physical lane count exceeds Helm release limit {limit} with {topology}" in refused.stderr From e6820804f1839e1c490adc637a5057c21c6eb4ce Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 05:18:11 +0100 Subject: [PATCH 05/40] fix(placement): price come-alive levers by planning again A come-alive lever's savings are now the fall in the plan total found by planning again from the plan the run wrote: savingAtPermittedBound with the identity's come-alive bound loosened to permittedBoundS, and savingUnderBound with the warmth's boot measured at the identity's bound. A lever is listed only when loosening its bound lowers the plan total. An identity that stays beside another resident of its current lane no longer claims the lane's floor as a saving: leaving the lane leaves that floor paid by the other residents, so loosening its bound raises the plan total and the planner offers no lever for it. --- packages/sie_placement/README.md | 14 +++- .../src/sie_placement/planner.py | 77 ++++++++++++++++-- .../fixtures/planner/expected-report.json | 40 ---------- packages/sie_placement/tests/test_levers.py | 79 ++++++++++++++++++- packages/sie_placement/tests/test_planner.py | 57 +++++++++++++ 5 files changed, 214 insertions(+), 53 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index cfeb962ed..72a4f4196 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -230,10 +230,16 @@ The report renders as a `placement-report` document: dropped, and the requests of an identity without a placement carry no bound and rank first. - `comeAliveLevers`: cold warmths whose measured come-alive misses the bound - on a cell that meets the warm bounds for less than the choice. Each is priced - under the identity's bound, where a bound of 0 admits no lever, and at - `permittedBoundS`, the bound its measured come-alive would permit. Levers rank - by the first saving, then by the second. + on a cell that meets the warm bounds for less than the choice, listed only + when loosening the identity's bound to `permittedBoundS`, the bound the + measured come-alive would permit, lowers the plan total. Both savings are + found by planning again from this plan and taking the fall in its total: + `savingAtPermittedBound` with the bound loosened, and `savingUnderBound` with + the warmth's boot measured at the identity's bound, which is 0 under a bound + of 0, since no boot comes under it, and when the total would not fall. An + identity that shares its lane saves nothing of the lane's floor by leaving, + since the other residents keep it. Levers rank by the saving under the bound, + then by the saving at the permitted bound. - `queueWaitUnattained`: every identity that declares `queueWaitP99Ms`. No plan claims that bound as met. - `refusals`, and `changes`: the identities and lanes that differ from diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index df445c623..2ca4306bd 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -337,6 +337,8 @@ def __init__( current_plan: Plan | None, parameters: PlannerParameters, ) -> None: + receipts = tuple(receipts) + boot_evidence = tuple(boot_evidence) placements = {placement.name: placement for placement in price_list.placements} placement = placements.get(parameters.placement) if placement is None: @@ -346,6 +348,10 @@ def __init__( f"the current plan is for placement {current_plan.placement!r}, not {parameters.placement!r}" ) self.parameters = parameters + self.service_levels = service_levels + self.receipt_records = receipts + self.boot_records = boot_evidence + self.demand = demand self.price_list = price_list self.multiplier = placement.region_multiplier self.prices = {profile.name: profile for profile in price_list.machine_profiles} @@ -524,8 +530,45 @@ def boot_requests(self, choice: _Choice) -> list[MeasurementRequest]: if chosen.cost - request.cost_bound > 0 ] - @staticmethod - def levers(choice: _Choice) -> list[Lever]: + def replanned_total( + self, + current: Plan, + *, + service_levels: ServiceLevels | None = None, + boot_evidence: Iterable[BootEvidence] | None = None, + ) -> float: + planner = _Planner( + self.service_levels if service_levels is None else service_levels, + self.receipt_records, + self.boot_records if boot_evidence is None else boot_evidence, + self.price_list, + self.demand, + current, + self.parameters, + ) + return planner.run(price_levers=False).report.monthly_cost + + def loosened(self, identity: str, bound: int) -> ServiceLevels: + return replace( + self.service_levels, + identities=tuple( + replace(entry, level=replace(entry.level, come_alive_p95_s=bound)) + if entry.identity == identity + else entry + for entry in self.service_levels.identities + ), + ) + + def booted_within(self, identity: str, option: _Option, bound: int) -> tuple[BootEvidence, ...]: + key = (identity, option.receipt.cell, option.receipt.container_shape, option.warmth) + return tuple( + replace(record, boot_seconds=min(record.boot_seconds, bound)) + if (record.identity, record.cell, record.container_shape, record.warmth) == key + else record + for record in self.boot_records + ) + + def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: chosen = choice.chosen bound = choice.level.come_alive_p95_s if chosen is None or bound is None: @@ -540,9 +583,22 @@ def levers(choice: _Choice) -> list[Lever]: known = best.get(key) if known is None or option.rank < known.rank: best[key] = option + savings: dict[int, float] = {} levers = [] for option in best.values(): - saving = chosen.cost - option.cost + permitted = max(1, math.ceil(option.come_alive_s)) + if permitted not in savings: + replanned = self.replanned_total(written, service_levels=self.loosened(choice.identity, permitted)) + savings[permitted] = _saving(total, replanned) + saving = savings[permitted] + if not saving: + continue + under = 0.0 + if bound > 0: + replanned = self.replanned_total( + written, boot_evidence=self.booted_within(choice.identity, option, bound) + ) + under = _saving(total, replanned) levers.append( Lever( identity=choice.identity, @@ -553,8 +609,8 @@ def levers(choice: _Choice) -> list[Lever]: come_alive_basis=option.basis, bound_s=bound, monthly_cost=option.cost, - saving_under_bound=saving if bound > 0 else 0.0, - permitted_bound_s=max(1, math.ceil(option.come_alive_s)), + saving_under_bound=under, + permitted_bound_s=permitted, saving_at_permitted_bound=saving, ) ) @@ -650,7 +706,7 @@ def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) - return window.above_floor_hours(requests, rate, lane.envelope.floor_min) return window.cold_container_hours(requests, rate) - def run(self) -> PlannerResult: + def run(self, *, price_levers: bool = True) -> PlannerResult: choices = [self.choose(identity) for identity in sorted(self.levels)] placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} kept = { @@ -678,13 +734,15 @@ def run(self) -> PlannerResult: receipts: dict[str, Receipt | None] = {identity: option.receipt for identity, option in placed.items()} receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) lane_reports, costs = self._pricing(new_plan, receipts) + total = sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) reports: list[IdentityReport] = [] requests: list[MeasurementRequest] = [] levers: list[Lever] = [] for choice in choices: requests.extend(self.receipt_requests(choice)) requests.extend(self.boot_requests(choice)) - levers.extend(self.levers(choice)) + if price_levers: + levers.extend(self.levers(choice, new_plan, total)) reports.append( self._identity_report(choice, names, kept, costs.get(choice.identity), receipts.get(choice.identity)) ) @@ -785,6 +843,11 @@ def _pricing( return tuple(reports), costs +def _saving(total: float, replanned: float) -> float: + saving = total - replanned + return saving if round(saving, 2) > 0 else 0.0 + + def _shared_chart_lanes(plan: Plan) -> tuple[SharedChartLane, ...]: grouped: defaultdict[tuple[str, str, str], list[str]] = defaultdict(list) for lane in plan.lanes: diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 7de1fe3db..e231c560c 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -534,46 +534,6 @@ "savingUnderBound": 379.6, "permittedBoundS": 45, "savingAtPermittedBound": 379.6 - }, - { - "identity": "BAAI/bge-m3", - "cell": { - "machineProfile": "l4", - "gpuProduct": "NVIDIA-L4" - }, - "containerShape": { - "gpuCount": 1, - "cpuCores": 4, - "memoryMiB": 16384 - }, - "warmth": "cold-with-bound", - "comeAliveS": 95.0, - "comeAliveBasis": "borrowed-boot-evidence", - "comeAliveBoundS": 0, - "monthlyCost": 569.4, - "savingUnderBound": 0.0, - "permittedBoundS": 95, - "savingAtPermittedBound": 189.8 - }, - { - "identity": "BAAI/bge-m3", - "cell": { - "machineProfile": "l4", - "gpuProduct": "NVIDIA-L4" - }, - "containerShape": { - "gpuCount": 1, - "cpuCores": 4, - "memoryMiB": 16384 - }, - "warmth": "snapshot-cold", - "comeAliveS": 8.5, - "comeAliveBasis": "boot-evidence", - "comeAliveBoundS": 0, - "monthlyCost": 569.4, - "savingUnderBound": 0.0, - "permittedBoundS": 9, - "savingAtPermittedBound": 189.8 } ], "queueWaitUnattained": [ diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index 9d762a98d..feed8adf7 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -13,11 +13,15 @@ Demand, DemandHour, DemandSeries, + Envelope, IdentityServiceLevel, MachineProfilePrice, Operation, Percentiles, Placement, + Plan, + PlanIdentity, + PlanLane, PriceList, Receipt, RequestShape, @@ -98,14 +102,39 @@ def busy(*identities, hours=6, requests=10): ) -def run(bounds, receipts, boots, demand=None): +def planned(bounds, receipts, boots, demand=None, current=None): service_levels = ServiceLevels( identities=tuple( IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) for identity, bound in bounds.items() ) ) - return plan(service_levels, receipts, boots, PRICES, demand or busy(*bounds), None, PARAMETERS).report + return plan(service_levels, receipts, boots, PRICES, demand or busy(*bounds), current, PARAMETERS) + + +def run(bounds, receipts, boots, demand=None): + return planned(bounds, receipts, boots, demand).report + + +def shared_lane(*identities): + return Plan( + placement="test", + lanes=( + PlanLane( + name="shared", + queue_pool="default", + bundle="default", + cell=L4, + container_shape=SHAPE, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=1, floor_max=1), + ), + ), + identities=tuple( + PlanIdentity(identity=identity, lane="shared", warmth=Warmth.PINNED) for identity in identities + ), + ) def cold_cost(hours=6): @@ -229,3 +258,49 @@ def test_a_lever_is_priced_at_the_cheapest_receipt_of_its_cell_and_shape(reverse (lever,) = report.come_alive_levers assert lever.monthly_cost == pytest.approx(L4_HOURLY * 6 * (1 / 0.02 + 1 * 2 * 900) / 3600 * SCALE) assert lever.saving_under_bound == pytest.approx(L4_HOURLY * MONTH - lever.monthly_cost) + + +@pytest.mark.parametrize(("bound", "seconds", "permitted"), [(0, 95.2, 96), (60, 240.0, 240)]) +def test_an_identity_staying_beside_a_neighbour_offers_no_lever_that_loosening_its_bound_would_not_pay( + bound, seconds, permitted +): + bounds = {"org/a": bound, "org/b": 0} + receipts = [receipt("org/a"), receipt("org/b")] + boots = [boot("org/a", seconds)] + result = planned(bounds, receipts, boots, current=shared_lane("org/a", "org/b")) + assert result.plan == shared_lane("org/a", "org/b") + assert result.report.identity("org/a").chosen.monthly_cost == pytest.approx(L4_HOURLY * MONTH) + assert result.report.come_alive_levers == () + loosened = planned({**bounds, "org/a": permitted}, receipts, boots, current=result.plan) + assert loosened.report.identity("org/a").chosen.warmth is Warmth.COLD_WITH_BOUND + assert loosened.report.monthly_cost - result.report.monthly_cost == pytest.approx(cold_cost()) + + +@pytest.mark.parametrize(("bound", "seconds", "permitted"), [(0, 95.2, 96), (60, 240.0, 240)]) +def test_an_identity_alone_on_its_current_lane_offers_the_lever_that_loosening_its_bound_pays( + bound, seconds, permitted +): + bounds = {"org/a": bound, "org/b": 0} + receipts = [receipt("org/a"), receipt("org/b")] + boots = [boot("org/a", seconds)] + result = planned(bounds, receipts, boots, current=shared_lane("org/a")) + assert result.report.identity("org/a").lane == "shared" + (lever,) = result.report.come_alive_levers + assert (lever.identity, lever.permitted_bound_s) == ("org/a", permitted) + loosened = planned({**bounds, "org/a": permitted}, receipts, boots, current=result.plan) + saving = result.report.monthly_cost - loosened.report.monthly_cost + assert saving == pytest.approx(L4_HOURLY * MONTH - cold_cost()) + assert lever.saving_at_permitted_bound == pytest.approx(saving) + assert lever.saving_under_bound == (0 if bound == 0 else pytest.approx(saving)) + + +def test_a_lever_saves_under_its_bound_what_its_own_boot_unlocks_and_at_its_permitted_bound_all_that_bound_unlocks(): + receipts = [receipt("org/model"), receipt("org/model", shape=WIDE)] + boots = [boot("org/model", 240.0), boot("org/model", 300.0, shape=WIDE)] + report = run({"org/model": 60}, receipts, boots) + by_shape = {lever.container_shape: lever for lever in report.come_alive_levers} + assert by_shape[WIDE].monthly_cost == pytest.approx((1.0 + 8 * 0.1) * 6 * 2 * SCALE) + assert by_shape[WIDE].saving_under_bound == pytest.approx(L4_HOURLY * MONTH - by_shape[WIDE].monthly_cost) + assert by_shape[WIDE].saving_at_permitted_bound == pytest.approx(L4_HOURLY * MONTH - cold_cost()) + assert by_shape[SHAPE].saving_under_bound == pytest.approx(L4_HOURLY * MONTH - cold_cost()) + assert [lever.container_shape for lever in report.come_alive_levers] == [SHAPE, WIDE] diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index c039599a0..d1065bbd1 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -4,6 +4,7 @@ import json import math import random +from dataclasses import replace from datetime import UTC, datetime, timedelta from pathlib import Path @@ -243,6 +244,16 @@ def rendered(result) -> tuple[str, str]: return render(result.plan.to_dict()), render(result.report.to_dict()) +def loosened(service_levels: ServiceLevels, identity: str, bound: int) -> ServiceLevels: + return replace( + service_levels, + identities=tuple( + replace(entry, level=replace(entry.level, come_alive_p95_s=bound)) if entry.identity == identity else entry + for entry in service_levels.identities + ), + ) + + def test_container_hourly_cost_prices_accelerators_cores_and_memory(): price = MachineProfilePrice( name="l4", gpu_product="NVIDIA-L4", accelerator_per_hour=2.0, core_per_hour=0.05, memory_gib_per_hour=0.01 @@ -1131,6 +1142,35 @@ def test_the_estate_report_states_its_findings(): ) +def test_an_estate_lever_saves_what_planning_again_at_its_permitted_bound_saves(): + inputs = estate() + result = plan(**inputs) + report = result.report + assert [(lever.identity, lever.warmth) for lever in report.come_alive_levers] == [ + ("Qwen/Qwen3-Embedding-4B", Warmth.COLD_WITH_BOUND), + ("Qwen/Qwen3-Embedding-4B", Warmth.SNAPSHOT_COLD), + ] + for lever in report.come_alive_levers: + service_levels = loosened(inputs["service_levels"], lever.identity, lever.permitted_bound_s) + again = plan(**{**inputs, "service_levels": service_levels, "current_plan": result.plan}) + assert report.monthly_cost - again.report.monthly_cost == pytest.approx(lever.saving_at_permitted_bound) + assert lever.saving_at_permitted_bound > 0 + + +def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever(): + inputs = estate() + result = plan(**inputs) + dense = result.report.identity("BAAI/bge-m3") + assert (dense.lane, dense.chosen.warmth) == ("default-l4-default", Warmth.PINNED) + assert dense.next_cheaper.come_alive_s == 8.5 + assert not [lever for lever in result.report.come_alive_levers if lever.identity == "BAAI/bge-m3"] + for bound in (9, 95): + service_levels = loosened(inputs["service_levels"], "BAAI/bge-m3", bound) + again = plan(**{**inputs, "service_levels": service_levels, "current_plan": result.plan}) + assert again.report.identity("BAAI/bge-m3").chosen.warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) + assert again.report.monthly_cost > result.report.monthly_cost + + @pytest.mark.parametrize("seed", range(25)) def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed): rng = random.Random(seed) # noqa: S311 -- reproducible test data @@ -1207,6 +1247,23 @@ def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed for lever in result.report.come_alive_levers: assert lever.monthly_cost < costs[lever.identity] assert lever.saving_at_permitted_bound >= lever.saving_under_bound >= 0 + assert lever.saving_at_permitted_bound > 0 + looser = loosened(service_levels, lever.identity, lever.permitted_bound_s) + again = run(looser, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) + assert result.report.monthly_cost - again.report.monthly_cost == pytest.approx(lever.saving_at_permitted_bound) + if lever.bound_s == 0: + assert lever.saving_under_bound == 0 + continue + key = (lever.identity, lever.cell, lever.container_shape, lever.warmth) + within = [ + replace(record, boot_seconds=min(record.boot_seconds, lever.bound_s)) + if (record.identity, record.cell, record.container_shape, record.warmth) == key + else record + for record in boots + ] + booted = run(service_levels, receipts, within, prices=prices, hourly=hourly, current=result.plan, **overrides) + fall = result.report.monthly_cost - booted.report.monthly_cost + assert lever.saving_under_bound == pytest.approx(max(0.0, fall), abs=0.01) for request in result.report.measurement_requests: assert request.saving_bound is None or request.saving_bound > 0 again = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) From 1c3d8c4e0f32e56b52f693d3b63db12fc8b662c0 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 05:40:35 +0100 Subject: [PATCH 06/40] feat(helm): add lane envelopes and an opt-in placement applier A worker lane may declare envelope.floorMin and envelope.floorMax, the warm-floor range a floor carrier may hold it at. The render fails unless floorMin equals minReplicas, floorMax lies between floorMin and maxReplicas, and, on an autoscaled lane with minReplicas 0, floorMax is 0 or at least the lane's fallback floor. An envelope renders nothing by itself. placementApplier.enabled renders one Deployment that runs `sie-placement tune` against the placement plan given in placementApplier.plan. It runs in shadow mode unless placementApplier.apply is true. Every plan lane must name an enabled worker lane whose envelope equals the plan lane's, so the plan cannot move a floor outside the range the chart values bound. The ScaledObject triggers are unchanged. With apply, the pod mounts the gateway admin token as a read-only file from the Secret named by gateway.auth.adminTokenSecretName; the chart never creates the token or renders its value. Pool reads use a client token from placementApplier.clientToken, which joins the token separation check. The pod has its own ServiceAccount with no token mounted and no RBAC, runs non-root with a read-only root filesystem, no privilege escalation, every capability dropped and the RuntimeDefault seccomp profile, and uses the Recreate strategy so one applier writes at a time. Literal gateway pool limits in gateway.extraEnv are passed to the applier. The applier is disabled by default and the default renders are unchanged. --- deploy/helm/sie-cluster/README.md | 136 ++++++- .../helm/sie-cluster/templates/_helpers.tpl | 173 ++++++++ .../templates/placement-applier.yaml | 182 +++++++++ deploy/helm/sie-cluster/values.yaml | 51 +++ tools/ci/tests/test_helm_render.py | 380 ++++++++++++++++++ 5 files changed, 919 insertions(+), 3 deletions(-) create mode 100644 deploy/helm/sie-cluster/templates/placement-applier.yaml diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 25e501533..36b46d307 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -490,6 +490,132 @@ metrics, the bundled collector Prometheus exporter, and its ServiceMonitor. The chart therefore cannot render collector-scoped application alerts without a scrape path. +### Lane envelopes and the placement applier + +A worker lane may declare an `envelope`, the range of warm floors that the +placement applier may hold it at: + +```yaml +workers: + pools: + l4: + enabled: true + bundles: + default: + minReplicas: 1 + maxReplicas: 4 + envelope: + floorMin: 1 + floorMax: 2 +``` + +The render fails unless `floorMin` and `floorMax` are non-negative integers, +`floorMin` equals `minReplicas`, and `floorMax` lies between `floorMin` and +`maxReplicas`. A lane returns to `minReplicas` whenever nothing raises its +floor, and KEDA never runs more than `maxReplicas`. With autoscaling enabled, a +lane with `minReplicas: 0` takes a `floorMax` of 0 or at least its fallback +floor, `min(maxReplicas, autoscaling.fallbackReplicas)`, because KEDA runs at +least that many workers once a floor activates the lane. The envelope renders +nothing by itself: the lane's StatefulSet and ScaledObject are the same with or +without it. + +`placementApplier.enabled=true` renders one Deployment that runs +`sie-placement tune` from the `sie-placement` package, whose README describes +the command, its audit rows and its refusals, against a placement plan: + +```bash +helm upgrade oci://ghcr.io/superlinked/charts/sie-cluster \ + --namespace \ + -f \ + --set placementApplier.enabled=true \ + --set placementApplier.image.repository= \ + --set placementApplier.image.tag= \ + --set-json "placementApplier.plan=$(cat plan.json)" +``` + +Every 60 seconds the applier reads each plan lane's series from the Prometheus +that KEDA queries, scoped to this release's collector target, reads the lane's +floor-carrier pool from the gateway, decides one step inside the lane's +envelope, and prints one JSON audit row per lane to its log. A floor carrier is +an API logical pool named `placement-floor---`. +The gateway keeps each lane at the largest warm floor among its pools and +reports it to the lane's warm-floor KEDA trigger, so a carrier raises a lane +above its chart floor and never lowers the floor another pool sets. The +ScaledObject triggers are the same with or without the applier. + +- **Shadow mode** (`placementApplier.apply: false`, the default) decides and + records and writes nothing. Run it first to see how often each lane reads as + under pressure. +- **Apply mode** (`placementApplier.apply: true`) upserts each lane's carrier on + every tick, which renews the carrier's lease and sets its floor. When the + applier stops, each carrier lapses 1200 seconds after its last write by + default, the gateway removes it, and the lane returns to its chart floor. + +The render fails unless: + +- `autoscaling.enabled=true`. +- `placementApplier.image.repository` and `placementApplier.image.tag` name an + image that installs `sie-placement` with its `keda` extra and can run as user + and group 65532. The chart publishes no such image. +- `placementApplier.plan` is a placement-plan document. Each of its lanes names + an enabled worker lane by its `queuePool`, `cell.machineProfile` and + `bundle`, compared as the chart canonicalizes lane tokens, and that worker + lane declares an `envelope` equal to the plan lane's. A plan therefore cannot + move a floor outside a range that the chart values bound. +- With gateway token auth, or a gateway auth mode known only at run time, + `placementApplier.clientToken.existingSecretName` names a Secret key that + holds one gateway client token, which the applier presents on pool reads. +- With `placementApplier.apply: true` and gateway token auth, or a mode known + only at run time, `gateway.auth.adminTokenSecretName` is set. +- `gateway.extraEnv` sets `SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT` and + `SIE_GATEWAY_MAX_POOLS`, if at all, as literal values. The chart passes an + integer value of either to the applier as `--max-pool-floor` or + `--max-pools`, so the applier refuses the plans the gateway would. + +The applier itself refuses a plan whose `floorMax` exceeds the gateway's +per-pool floor limit, 4 by default, or whose lanes need more carriers than the +gateway admits, 64 by default. A refusal, an unreadable token file or an +invalid flag ends the container, which then shows as `CrashLoopBackOff`. A +failed Prometheus query or pool call fails only its tick, which the audit row +records. + +#### Applier permissions + +| Credential | Held | Authorizes | +| --- | --- | --- | +| Gateway admin token | Only with `placementApplier.apply: true`, from the Secret `gateway.auth.adminTokenSecretName` (key `gateway.auth.adminTokenSecretKey`) | `POST`, `PUT` and `DELETE` under `/v1/pools`, `/v1/admin` and `/v1/configs`: every gateway admin route, not only floor carriers | +| Gateway client token | When `placementApplier.clientToken.existingSecretName` is set | Every gateway route that is not an admin route, inference included; the applier uses it for `GET /v1/pools/{name}` | +| Kubernetes API | Never | Nothing: the applier has its own ServiceAccount, mounts no ServiceAccount token, and has no Role or RoleBinding | + +The chart never creates either token or renders its value. Each is mounted +read-only as a file of mode `0440` owned by group 65532, under +`/etc/sie-placement/`, and passed by path with `--admin-token-file` or +`--api-key-file`. The applier sends the admin token only on pool writes to the +in-cluster gateway Service, never the client token on writes, and never takes a +token on its command line or prints or logs one: a rejected token file is named +by its path, and a failed gateway call by the server's error message or the +exception type. Anyone who can read the applier's log sees lane telemetry and +floors, not credentials. Anyone who can create pods in the namespace with that +Secret can still read it, so the Secret read limits under +[Who holds which token](#who-holds-which-token) apply. + +The Deployment runs one replica with the `Recreate` strategy, so at most one +applier writes carriers, also during a rollout. Its pod runs as user and group +65532 with `runAsNonRoot`, the `RuntimeDefault` seccomp profile, a read-only +root filesystem, no privilege escalation and every capability dropped. A +change to `placementApplier.plan` restarts it. + +#### Applier network access + +The applier accepts no connections. It opens connections to the gateway +Service (`gateway.service.port`, 8080 by default), to the Prometheus query API +(`autoscaling.prometheusAddress`, or the bundled Prometheus Service), and to +cluster DNS. The chart renders no NetworkPolicy for it. Under a default-deny +egress policy, allow those destinations for pods labelled +`app.kubernetes.io/component: placement-applier` and +`app.kubernetes.io/instance: `. A policy that restricts ingress to the +gateway or to Prometheus must admit the same pods. + ### Worker-sidecar Telemetry When `workers.common.workerSidecar.enabled=true`, each worker Pod includes a @@ -1412,7 +1538,9 @@ default `SIE_ADMIN_TOKEN`), which the chart passes to the gateway as configured`). The chart does not generate this token, and it must be separate from the sie-config tokens: the render fails when it names the Secret and key of the sie-config admin token, which would give every gateway pod catalog -write access, or of the read token, which every worker sidecar holds. +write access, or of the read token, which every worker sidecar holds. The +placement applier mounts the same Secret only when `placementApplier.apply` is +true. ### Who holds which token @@ -1420,8 +1548,10 @@ write access, or of the read token, which every worker sidecar holds. - **sie-config read token:** sie-config, every gateway pod, and every worker sidecar. It can read the whole catalog, including every model config, but cannot write. -- **Gateway admin token (when set):** every gateway pod and the tooling that - manages pools. +- **Gateway admin token (when set):** every gateway pod, the placement applier + when `placementApplier.apply` is true (see + [Lane envelopes and the placement applier](#lane-envelopes-and-the-placement-applier)), + and the tooling that manages pools. - **Gateway client token (when set or generated):** API clients, every gateway pod, and every worker sidecar, which presents it when it calls the gateway. See [Securing the cluster](#securing-the-cluster). diff --git a/deploy/helm/sie-cluster/templates/_helpers.tpl b/deploy/helm/sie-cluster/templates/_helpers.tpl index f86c05e32..40195b521 100644 --- a/deploy/helm/sie-cluster/templates/_helpers.tpl +++ b/deploy/helm/sie-cluster/templates/_helpers.tpl @@ -1051,6 +1051,118 @@ every tool call may be refused. {{- end -}} {{- end }} +{{- define "sie-cluster.placementApplier.enabled" -}} +{{- include "sie-cluster.optIn" (dict "value" (dig "enabled" nil (.Values.placementApplier | default dict)) "path" "placementApplier.enabled") -}} +{{- end }} + +{{- define "sie-cluster.placementApplier.apply" -}} +{{- include "sie-cluster.optIn" (dict "value" (dig "apply" nil (.Values.placementApplier | default dict)) "path" "placementApplier.apply") -}} +{{- end }} + +{{- define "sie-cluster.placementApplier.name" -}} +{{- printf "%s-placement-applier" (include "sie-cluster.fullname" .) -}} +{{- end }} + +{{- define "sie-cluster.placementApplier.selectorLabels" -}} +{{ include "sie-cluster.selectorLabels" . }} +app.kubernetes.io/component: placement-applier +{{- end }} + +{{/* +Validation: the placement applier needs KEDA autoscaling, an image, a gateway +client token when the gateway may authenticate, the gateway admin token Secret +when it writes, and a placement plan whose lanes all match enabled worker +lanes and their envelopes. +*/}} +{{- define "sie-cluster.placementApplier.validate" -}} +{{- $root := . -}} +{{- $applier := $root.Values.placementApplier | default dict -}} +{{- if not $root.Values.autoscaling.enabled -}} +{{- fail "placementApplier.enabled needs autoscaling.enabled: the applier raises a lane's floor through the warm-floor trigger of the lane's KEDA ScaledObject and reads the lane series that autoscaling exports" -}} +{{- end -}} +{{- $image := $applier.image | default dict -}} +{{- if not (and (trim (toString (default "" $image.repository))) (trim (toString (default "" $image.tag)))) -}} +{{- fail "placementApplier.image.repository and placementApplier.image.tag are required: the chart publishes no image that runs sie-placement. Use an image that installs sie-placement with its keda extra." -}} +{{- end -}} +{{- $clientToken := $applier.clientToken | default dict -}} +{{- if and $clientToken.existingSecretName (not (trim (toString (default "" $clientToken.secretKey)))) -}} +{{- fail "placementApplier.clientToken.secretKey is empty. Set it to the key of placementApplier.clientToken.existingSecretName that holds the gateway client token." -}} +{{- end -}} +{{- $auth := include "sie-cluster.gateway.effectiveAuthMode" $root | fromJson -}} +{{- $mayAuthenticate := or (include "sie-cluster.gateway.authenticates" $root) (not $auth.known) -}} +{{- if and $mayAuthenticate (not $clientToken.existingSecretName) -}} +{{- $example := "" -}} +{{- with include "sie-cluster.gateway.tokenSecretName" $root -}} +{{- $example = printf ", for example placementApplier.clientToken.existingSecretName=%s and placementApplier.clientToken.secretKey=%s" . (include "sie-cluster.gateway.tokenSecretKey" $root) -}} +{{- end -}} +{{- fail (printf "placementApplier.enabled with gateway token auth (or a gateway auth mode known only at run time) needs placementApplier.clientToken.existingSecretName: the applier reads pools from the gateway, which refuses requests without a token. Point it at a Secret key that holds one gateway client token%s." $example) -}} +{{- end -}} +{{- if and (include "sie-cluster.placementApplier.apply" $root) $mayAuthenticate (not (dig "auth" "adminTokenSecretName" "" ($root.Values.gateway | default dict))) -}} +{{- fail "placementApplier.apply=true with gateway token auth (or a gateway auth mode known only at run time) needs gateway.auth.adminTokenSecretName: the gateway refuses pool writes without its admin token, which the applier reads from that Secret." -}} +{{- end -}} +{{- if hasKey ($applier.podAnnotations | default dict) "checksum/plan" -}} +{{- fail "placementApplier.podAnnotations must not override chart-owned annotation checksum/plan" -}} +{{- end -}} +{{- include "sie-cluster.placementApplier.validatePlan" $root -}} +{{- end }} + +{{/* +Validation: every lane of placementApplier.plan names an enabled worker lane by +its canonical queuePool, cell.machineProfile and bundle, and that worker lane +declares an envelope equal to the plan lane's, so the plan cannot move a floor +outside the range the chart values bound. +*/}} +{{- define "sie-cluster.placementApplier.validatePlan" -}} +{{- $root := . -}} +{{- $plan := dig "plan" nil ($root.Values.placementApplier | default dict) -}} +{{- if not (and (kindIs "map" $plan) (eq (toString (index $plan "kind")) "placement-plan")) -}} +{{- fail "placementApplier.plan must be a placement-plan document, for example --set-json \"placementApplier.plan=$(cat plan.json)\"" -}} +{{- end -}} +{{- $lanes := index $plan "lanes" -}} +{{- if not (and (kindIs "slice" $lanes) $lanes) -}} +{{- fail "placementApplier.plan.lanes must list at least one lane" -}} +{{- end -}} +{{- $chartLanes := dict -}} +{{- range $poolName, $pool := $root.Values.workers.pools -}} +{{- if $pool.enabled -}} +{{- range $bundleName, $bundleCfg := $pool.bundles -}} +{{- if dig "enabled" true $bundleCfg -}} +{{- $lane := include "sie-cluster.worker.lane" (dict "root" $root "poolName" $poolName "pool" $pool "bundleName" $bundleName) | fromJson -}} +{{- $_ := set $chartLanes (printf "%s|%s|%s" $lane.queuePool $lane.machineProfile $lane.bundle) (dict "ownerPath" $lane.ownerPath "envelope" $bundleCfg.envelope) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range $index, $planLane := $lanes -}} +{{- $path := printf "placementApplier.plan.lanes[%d]" $index -}} +{{- if not (and (kindIs "map" $planLane) (kindIs "map" (index $planLane "cell")) (kindIs "map" (index $planLane "envelope"))) -}} +{{- fail (printf "%s must be a plan lane with a cell and an envelope" $path) -}} +{{- end -}} +{{- $tokens := list (index $planLane "queuePool") (index $planLane "cell" "machineProfile") (index $planLane "bundle") -}} +{{- $key := list -}} +{{- range $token := $tokens -}} +{{- $key = append $key (default "" $token | toString | trim | lower) -}} +{{- end -}} +{{- $key = join "|" $key -}} +{{- $chart := index $chartLanes $key -}} +{{- if not $chart -}} +{{- fail (printf "%s names the lane %q (queuePool|machineProfile|bundle), which is not an enabled worker lane" $path $key) -}} +{{- end -}} +{{- if not $chart.envelope -}} +{{- fail (printf "%s names %s, which declares no envelope: set %s.envelope to the plan lane's floorMin and floorMax" $path $chart.ownerPath $chart.ownerPath) -}} +{{- end -}} +{{- $planEnvelope := index $planLane "envelope" -}} +{{- $bounds := dict -}} +{{- range $field := list "floorMin" "floorMax" -}} +{{- $_ := set $bounds (printf "plan%s" $field) (include "sie-cluster.worker.envelopeBound" (dict "path" (printf "%s.envelope.%s" $path $field) "value" (index $planEnvelope $field))) -}} +{{- $_ = set $bounds (printf "chart%s" $field) (include "sie-cluster.worker.envelopeBound" (dict "path" (printf "%s.envelope.%s" $chart.ownerPath $field) "value" (index $chart.envelope $field))) -}} +{{- end -}} +{{- if or (ne $bounds.planfloorMin $bounds.chartfloorMin) (ne $bounds.planfloorMax $bounds.chartfloorMax) -}} +{{- fail (printf "%s.envelope (floorMin %s, floorMax %s) differs from %s.envelope (floorMin %s, floorMax %s): the worker lane's envelope bounds the floor the applier may write" $path $bounds.planfloorMin $bounds.planfloorMax $chart.ownerPath $bounds.chartfloorMin $bounds.chartfloorMax) -}} +{{- end -}} +{{- end -}} +{{- end }} + {{/* Validation: workers.auth.allowUnauthenticated is the first step of the two-step upgrade and needs workers.auth.enabled. @@ -1243,6 +1355,12 @@ token-separation check and the upstream-key collision check read this list. {{- with $mcp.clusterApiKey.existingSecretName -}} {{- $keys = append $keys (dict "owner" "mcpEdge.clusterApiKey.existingSecretName" "name" "MCP edge cluster API key" "ref" (printf "%s/%s" . $mcp.clusterApiKey.secretKey) "role" "gateway-client") -}} {{- end -}} +{{- if include "sie-cluster.placementApplier.enabled" $root -}} +{{- $applierToken := dig "clientToken" dict ($root.Values.placementApplier | default dict) -}} +{{- with $applierToken.existingSecretName -}} +{{- $keys = append $keys (dict "owner" "placementApplier.clientToken.existingSecretName" "name" "placement applier client token" "ref" (printf "%s/%s" . $applierToken.secretKey) "role" "gateway-client") -}} +{{- end -}} +{{- end -}} {{- $tls := include "sie-cluster.ingressTlsConfig" $root | fromYaml -}} {{- $tlsSecrets := list (dict "owner" "the ingress TLS certificate" "name" $tls.secretName) (dict "owner" "the MCP ingress TLS certificate" "name" (printf "%s-tls" (include "sie-cluster.mcpEdge.serviceName" $root))) -}} {{- $rootCA := dig "selfSigned" "rootCA" dict $tls -}} @@ -1682,8 +1800,63 @@ consume the same queuePool/machineProfile/bundle tuple. {{- fail (printf "%s: canonical physical lane %q duplicates %s after trim/lower normalization" $lane.ownerPath $key (index $seen $key)) -}} {{- end -}} {{- $_ := set $seen $key $lane.ownerPath -}} +{{- if $bundleCfg.envelope -}} +{{- $_ = include "sie-cluster.worker.validateEnvelope" (dict "root" $root "lane" $lane "bundle" $bundleCfg) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +A non-negative integer bound read from values or from a placement plan, +returned in canonical decimal form. Args (dict): path, value. +*/}} +{{- define "sie-cluster.worker.envelopeBound" -}} +{{- $value := .value -}} +{{- $text := toString $value -}} +{{- if not (and (or (kindIs "float64" $value) (kindIs "int" $value) (kindIs "int64" $value)) (regexMatch "^(0|[1-9][0-9]*)$" $text)) -}} +{{- fail (printf "%s must be a non-negative integer, got %s" .path (ternary "nothing" $text (kindIs "invalid" $value))) -}} +{{- end -}} +{{- $text -}} +{{- end }} + +{{/* +Validate a worker lane's envelope: the warm-floor range a floor carrier may +hold the lane at. A lane returns to minReplicas when no carrier holds it, KEDA +runs at most maxReplicas, and an autoscaled lane with minReplicas 0 runs at +least its fallback floor once a carrier activates it. +Args (dict): root, lane (sie-cluster.worker.lane), bundle. +*/}} +{{- define "sie-cluster.worker.validateEnvelope" -}} +{{- $path := printf "%s.envelope" .lane.ownerPath -}} +{{- $envelope := .bundle.envelope -}} +{{- if not (kindIs "map" $envelope) -}} +{{- fail (printf "%s must be a map with floorMin and floorMax" $path) -}} +{{- end -}} +{{- range $field := keys $envelope | sortAlpha -}} +{{- if not (has $field (list "floorMin" "floorMax")) -}} +{{- fail (printf "%s.%s is not an envelope field: a worker lane's envelope holds floorMin and floorMax only" $path $field) -}} +{{- end -}} +{{- end -}} +{{- $floorMin := int (include "sie-cluster.worker.envelopeBound" (dict "path" (printf "%s.floorMin" $path) "value" (index $envelope "floorMin"))) -}} +{{- $floorMax := int (include "sie-cluster.worker.envelopeBound" (dict "path" (printf "%s.floorMax" $path) "value" (index $envelope "floorMax"))) -}} +{{- $minReplicas := int .bundle.minReplicas -}} +{{- $maxReplicas := int .bundle.maxReplicas -}} +{{- if ne $floorMin $minReplicas -}} +{{- fail (printf "%s.floorMin must equal minReplicas (%d), got %d: the lane returns to minReplicas whenever no floor carrier holds it" $path $minReplicas $floorMin) -}} +{{- end -}} +{{- if lt $floorMax $floorMin -}} +{{- fail (printf "%s.floorMax must be at least floorMin (%d), got %d" $path $floorMin $floorMax) -}} {{- end -}} +{{- if gt $floorMax $maxReplicas -}} +{{- fail (printf "%s.floorMax must not exceed maxReplicas (%d), got %d: KEDA runs at most maxReplicas" $path $maxReplicas $floorMax) -}} {{- end -}} +{{- if and .root.Values.autoscaling.enabled (eq $floorMin 0) (gt $floorMax 0) -}} +{{- $fallbackFloor := min $maxReplicas (max $minReplicas (int .root.Values.autoscaling.fallbackReplicas)) -}} +{{- if lt $floorMax $fallbackFloor -}} +{{- fail (printf "%s.floorMax must be 0 or at least the lane's fallback floor (%d), got %d: a raised floor on a lane with minReplicas 0 runs min(maxReplicas, autoscaling.fallbackReplicas) workers or more" $path $fallbackFloor $floorMax) -}} {{- end -}} {{- end -}} {{- end }} diff --git a/deploy/helm/sie-cluster/templates/placement-applier.yaml b/deploy/helm/sie-cluster/templates/placement-applier.yaml new file mode 100644 index 000000000..357a068cd --- /dev/null +++ b/deploy/helm/sie-cluster/templates/placement-applier.yaml @@ -0,0 +1,182 @@ +{{- if include "sie-cluster.placementApplier.enabled" . }} +{{- $_ := include "sie-cluster.worker.validateLanes" . -}} +{{- include "sie-cluster.placementApplier.validate" . -}} +{{- $applier := .Values.placementApplier -}} +{{- $apply := include "sie-cluster.placementApplier.apply" . -}} +{{- $name := include "sie-cluster.placementApplier.name" . -}} +{{- $planJson := toPrettyJson $applier.plan -}} +{{- $promAddr := .Values.autoscaling.prometheusAddress -}} +{{- if index .Values "kube-prometheus-stack" "install" -}} +{{- $promAddr = include "sie-cluster.kubePrometheusStack.prometheusAddress" . -}} +{{- end -}} +{{- $seriesSelector := printf `namespace="%s",service="%s-otel-collector",endpoint="prometheus"` (include "sie-cluster.namespace" .) (include "sie-cluster.fullname" .) -}} +{{- $poolLimitArgs := list -}} +{{- range $setting := list (dict "env" "SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT" "flag" "--max-pool-floor") (dict "env" "SIE_GATEWAY_MAX_POOLS" "flag" "--max-pools") -}} +{{- $value := "" -}} +{{- range $entry := (default (list) $.Values.gateway.extraEnv) -}} +{{- if and (kindIs "map" $entry) (eq (toString (index $entry "name")) $setting.env) -}} +{{- if not (hasKey $entry "value") -}} +{{- fail (printf "placementApplier.enabled needs gateway.extraEnv %s as a literal value, so the applier refuses the plans the gateway would refuse" $setting.env) -}} +{{- end -}} +{{- $value = index $entry "value" | toString | trim -}} +{{- end -}} +{{- end -}} +{{- if regexMatch "^[0-9]+$" $value -}} +{{- $poolLimitArgs = append $poolLimitArgs (dict "flag" $setting.flag "value" $value) -}} +{{- end -}} +{{- end -}} +{{- $clientToken := $applier.clientToken | default dict -}} +{{- $adminTokenSecret := "" -}} +{{- if $apply -}} +{{- $adminTokenSecret = dig "auth" "adminTokenSecretName" "" (.Values.gateway | default dict) -}} +{{- end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ include "sie-cluster.namespace" . }} + labels: + {{- include "sie-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: placement-applier +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-plan" $name }} + namespace: {{ include "sie-cluster.namespace" . }} + labels: + {{- include "sie-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: placement-applier +data: + plan.json: | + {{- $planJson | nindent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + namespace: {{ include "sie-cluster.namespace" . }} + labels: + {{- include "sie-cluster.labels" . | nindent 4 }} + app.kubernetes.io/component: placement-applier +spec: + {{- /* One writer of the floor carriers at a time, also during a rollout. */}} + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "sie-cluster.placementApplier.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "sie-cluster.placementApplier.selectorLabels" . | nindent 8 }} + annotations: + {{- with $applier.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/plan: {{ $planJson | sha256sum | quote }} + spec: + {{- include "sie-cluster.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ $name }} + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: applier + image: {{ printf "%s:%s" (trim (toString $applier.image.repository)) (trim (toString $applier.image.tag)) | quote }} + imagePullPolicy: {{ $applier.image.pullPolicy | default "IfNotPresent" }} + command: ["sie-placement"] + args: + - tune + - --plan + - /etc/sie-placement/plan/plan.json + - --prometheus-url + - {{ $promAddr | quote }} + - --gateway-url + - {{ include "sie-cluster.gateway.internalUrl" . | quote }} + - --series-selector + - {{ $seriesSelector | quote }} + {{- range $poolLimitArgs }} + - {{ .flag }} + - {{ .value | quote }} + {{- end }} + {{- if $clientToken.existingSecretName }} + - --api-key-file + - /etc/sie-placement/client-token/token + {{- end }} + {{- if $apply }} + - --apply + {{- if $adminTokenSecret }} + - --admin-token-file + - /etc/sie-placement/admin-token/token + {{- end }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + {{- toYaml $applier.resources | nindent 12 }} + volumeMounts: + - name: plan + mountPath: /etc/sie-placement/plan + readOnly: true + {{- if $clientToken.existingSecretName }} + - name: client-token + mountPath: /etc/sie-placement/client-token + readOnly: true + {{- end }} + {{- if $adminTokenSecret }} + - name: admin-token + mountPath: /etc/sie-placement/admin-token + readOnly: true + {{- end }} + volumes: + - name: plan + configMap: + name: {{ printf "%s-plan" $name }} + items: + - key: plan.json + path: plan.json + {{- with $clientToken.existingSecretName }} + - name: client-token + secret: + secretName: {{ . }} + defaultMode: 0440 + items: + - key: {{ $clientToken.secretKey }} + path: token + {{- end }} + {{- with $adminTokenSecret }} + - name: admin-token + secret: + secretName: {{ . }} + defaultMode: 0440 + items: + - key: {{ include "sie-cluster.gateway.adminTokenSecretKey" $ }} + path: token + {{- end }} + {{- with $applier.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $applier.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $applier.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index 5c8b979a0..f15612fc9 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -1143,6 +1143,14 @@ workers: # bundle-level entry wins on duplicate names. # maxReplicas: KEDA upper bound. # minReplicas: baseline replicas (0 = scale-to-zero). + # envelope: Optional `{floorMin, floorMax}`, the warm-floor range + # that placementApplier may hold this lane at. floorMin + # must equal minReplicas, and floorMax must lie between + # floorMin and maxReplicas. With autoscaling enabled and + # minReplicas 0, a floorMax above 0 must be at least the + # lane's fallback floor, min(maxReplicas, + # autoscaling.fallbackReplicas). See "Lane envelopes and + # the placement applier" in the chart README. # # Each queuePool/machineProfile/bundle lane renders one StatefulSet + # KEDA ScaledObject + PDB + image-prepull DaemonSet, named worker-{pool}-{bundle}. @@ -1390,6 +1398,49 @@ autoscaling: # broken metrics path to race the fallback floor back to zero. fallbackReplicas: 2 +# -- Placement applier: one Deployment that runs `sie-placement tune` from the +# sie-placement package to hold each lane of a placement plan at a warm floor +# inside its envelope, through one gateway floor-carrier pool per lane. +# Requires autoscaling.enabled. See "Lane envelopes and the placement applier" +# in the chart README. +placementApplier: + # -- Render the applier (a boolean). + enabled: false + # -- Write the floor-carrier pools (a boolean). false runs in shadow mode: + # the applier decides and prints one audit row per lane and writes nothing. + # true also mounts the gateway admin token from the Secret named by + # gateway.auth.adminTokenSecretName, which authorizes every gateway admin + # route, and passes --apply. + apply: false + # -- An image that runs `sie-placement` with the sie-placement keda extra + # installed, as a non-root user. The chart publishes none, so repository and + # tag are required. + image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + # -- The placement-plan document, as a map, for example + # `--set-json "placementApplier.plan=$(cat plan.json)"`. Every plan lane must + # name an enabled worker lane whose envelope equals the plan lane's. + plan: {} + # -- Gateway client token for pool reads, from an existing Secret. Required + # when the gateway enforces token auth or its auth mode is known only at run + # time. The key must hold one token. + clientToken: + existingSecretName: "" + secretKey: api-key + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + # -- Ingress configuration ingress: enabled: false diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 15db7f278..5a667bac3 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import hashlib import json import os import re @@ -4212,3 +4213,382 @@ def test_placement_values_at_the_lane_limit_render( refused = render_placement(tmp_path, base, placement_lane_values(limit - planned + 1), overlay) assert refused.returncode != 0 assert f"configured physical lane count exceeds Helm release limit {limit} with {topology}" in refused.stderr + + +APPLIER = "sie-sie-cluster-placement-applier" +APPLIER_IMAGE = {"repository": "registry.example.com/sie-placement", "tag": "1.0.0"} +APPLIER_ARGS = [ + "tune", + "--plan", + "/etc/sie-placement/plan/plan.json", + "--prometheus-url", + "http://prometheus-operated.monitoring.svc.cluster.local:9090", + "--gateway-url", + "http://sie-sie-cluster-gateway.sie.svc.cluster.local:8080", + "--series-selector", + 'namespace="sie",service="sie-sie-cluster-otel-collector",endpoint="prometheus"', +] +APPLIER_AUTH = { + "gateway": { + "auth": { + "mode": "token", + "tokenSecretName": "gateway-tokens", + "adminTokenSecretName": "gateway-admin", + "adminTokenSecretKey": "admin", + } + } +} +APPLIER_CLIENT_TOKEN = {"existingSecretName": "applier-client", "secretKey": "token"} +SECRET_FILE_MODE = 0o440 + + +def placement_plan() -> dict: + return json.loads((PLACEMENT_FIXTURES / "plan.json").read_text(encoding="utf-8")) + + +def plan_envelopes(overlay: dict, plan: dict) -> dict: + lanes = placement_lanes(overlay) + pools: dict = {} + for lane in plan["lanes"]: + key = tuple( + token.strip().lower() for token in (lane["queuePool"], lane["cell"]["machineProfile"], lane["bundle"]) + ) + group, bundle, _ = lanes[key] + envelope = {field: lane["envelope"][field] for field in ("floorMin", "floorMax")} + pools.setdefault(group, {"bundles": {}})["bundles"][bundle] = {"envelope": envelope} + return {"workers": {"pools": pools}} + + +def applier_layers(*extra: dict, **applier: object) -> list[dict]: + overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) + plan = placement_plan() + settings = {"enabled": True, "image": APPLIER_IMAGE, "plan": plan, **applier} + return [AUTOSCALING_VALUES, overlay, plan_envelopes(overlay, plan), *extra, {"placementApplier": settings}] + + +def placement_documents(tmp_path: Path, layers: list[dict]) -> list[dict]: + result = render_placement(tmp_path, *layers) + assert result.returncode == 0, result.stderr + return [doc for doc in yaml.safe_load_all(result.stdout) if doc] + + +def applier_documents(docs: list[dict]) -> list[dict]: + return [ + doc + for doc in docs + if doc["metadata"].get("labels", {}).get("app.kubernetes.io/component") == "placement-applier" + ] + + +def applier_pod(docs: list[dict]) -> dict: + (deployment,) = [doc for doc in docs if doc["kind"] == "Deployment" and doc["metadata"]["name"] == APPLIER] + return deployment["spec"]["template"] + + +def without_secrets(docs: list[dict]) -> list[dict]: + return [doc for doc in docs if doc["kind"] != "Secret"] + + +@pytest.mark.parametrize("values", [{}, {"placementApplier": None}]) +def test_the_placement_applier_is_off_by_default(tmp_path: Path, values: dict) -> None: + assert applier_documents(rendered_documents(tmp_path, values)) == [] + overlay = json.loads(PLACEMENT_VALUES.read_text(encoding="utf-8")) + planned = placement_documents(tmp_path, [AUTOSCALING_VALUES, overlay]) + with_envelopes = placement_documents( + tmp_path, [AUTOSCALING_VALUES, overlay, plan_envelopes(overlay, placement_plan())] + ) + assert applier_documents(with_envelopes) == [] + assert without_secrets(with_envelopes) == without_secrets(planned) + + +def test_the_placement_applier_runs_in_shadow_mode_without_credentials(tmp_path: Path) -> None: + docs = placement_documents(tmp_path, applier_layers()) + assert sorted((doc["kind"], doc["metadata"]["name"]) for doc in applier_documents(docs)) == [ + ("ConfigMap", f"{APPLIER}-plan"), + ("Deployment", APPLIER), + ("ServiceAccount", APPLIER), + ] + (config_map,) = [doc for doc in docs if doc["kind"] == "ConfigMap" and doc["metadata"]["name"] == f"{APPLIER}-plan"] + plan_json = config_map["data"]["plan.json"] + assert json.loads(plan_json) == placement_plan() + (account,) = [doc for doc in docs if doc["kind"] == "ServiceAccount" and doc["metadata"]["name"] == APPLIER] + assert account["automountServiceAccountToken"] is False + assert "annotations" not in account["metadata"] + assert not [ + doc + for doc in docs + if doc["kind"] in {"RoleBinding", "ClusterRoleBinding"} + and any(subject.get("name") == APPLIER for subject in doc.get("subjects", [])) + ] + + (deployment,) = [doc for doc in docs if doc["kind"] == "Deployment" and doc["metadata"]["name"] == APPLIER] + assert deployment["spec"]["replicas"] == 1 + assert deployment["spec"]["strategy"] == {"type": "Recreate"} + pod = applier_pod(docs) + assert pod["metadata"]["annotations"] == { + "checksum/plan": hashlib.sha256(plan_json.rstrip("\n").encode()).hexdigest() + } + spec = pod["spec"] + assert spec["serviceAccountName"] == APPLIER + assert spec["automountServiceAccountToken"] is False + assert spec["securityContext"] == { + "runAsNonRoot": True, + "runAsUser": 65532, + "runAsGroup": 65532, + "fsGroup": 65532, + "seccompProfile": {"type": "RuntimeDefault"}, + } + (container,) = spec["containers"] + assert container["image"] == "registry.example.com/sie-placement:1.0.0" + assert container["command"] == ["sie-placement"] + assert container["args"] == APPLIER_ARGS + assert "env" not in container + assert container["securityContext"] == { + "allowPrivilegeEscalation": False, + "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}, + } + assert container["volumeMounts"] == [{"name": "plan", "mountPath": "/etc/sie-placement/plan", "readOnly": True}] + assert spec["volumes"] == [ + {"name": "plan", "configMap": {"name": f"{APPLIER}-plan", "items": [{"key": "plan.json", "path": "plan.json"}]}} + ] + + +@pytest.mark.parametrize("apply", [False, True]) +def test_the_placement_applier_mounts_the_admin_token_only_to_write(tmp_path: Path, apply: bool) -> None: + docs = placement_documents(tmp_path, applier_layers(APPLIER_AUTH, apply=apply, clientToken=APPLIER_CLIENT_TOKEN)) + spec = applier_pod(docs)["spec"] + (container,) = spec["containers"] + client = ["--api-key-file", "/etc/sie-placement/client-token/token"] + write = ["--apply", "--admin-token-file", "/etc/sie-placement/admin-token/token"] if apply else [] + assert container["args"] == [*APPLIER_ARGS, *client, *write] + mounts = {mount["name"]: mount for mount in container["volumeMounts"]} + volumes = {volume["name"]: volume for volume in spec["volumes"]} + expected = {"client-token": ("applier-client", "token")} + if apply: + expected["admin-token"] = ("gateway-admin", "admin") + assert set(mounts) == set(volumes) == {"plan", *expected} + for name, (secret, key) in expected.items(): + assert mounts[name] == {"name": name, "mountPath": f"/etc/sie-placement/{name}", "readOnly": True} + assert volumes[name]["secret"] == { + "secretName": secret, + "defaultMode": SECRET_FILE_MODE, + "items": [{"key": key, "path": "token"}], + } + assert "env" not in container + assert not [ + doc + for doc in docs + if doc["kind"] == "Secret" and doc["metadata"]["name"] in {"gateway-admin", "applier-client"} + ] + assert env_entries(docs, "SIE_ADMIN_TOKEN") == { + CONFIG_SERVICE: GENERATED_ADMIN, + GATEWAY: {"name": "gateway-admin", "key": "admin"}, + } + + +def test_the_placement_applier_uses_the_gateways_literal_pool_limits(tmp_path: Path) -> None: + extra_env = [ + {"name": "SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT", "value": " 6 "}, + {"name": "SIE_GATEWAY_MAX_POOLS", "value": "many"}, + ] + docs = placement_documents(tmp_path, applier_layers({"gateway": {"extraEnv": extra_env}})) + (container,) = applier_pod(docs)["spec"]["containers"] + assert container["args"] == [*APPLIER_ARGS, "--max-pool-floor", "6"] + + +def drop_l4_envelope(layers: list[dict]) -> None: + del layers[2]["workers"]["pools"]["l4"]["bundles"]["default"]["envelope"] + + +def widen_l4_envelope(layers: list[dict]) -> None: + layers[2]["workers"]["pools"]["l4"]["bundles"]["default"]["envelope"]["floorMax"] = 3 + + +def disable_h100(layers: list[dict]) -> None: + layers[1]["workers"]["pools"]["h100"]["enabled"] = False + + +def plan_without_lanes(layers: list[dict]) -> None: + layers[-1]["placementApplier"]["plan"]["lanes"] = [] + + +@pytest.mark.parametrize( + ("change", "applier", "error"), + [ + ( + lambda layers: layers.__setitem__(0, {}), + {}, + "placementApplier.enabled needs autoscaling.enabled", + ), + (None, {"image": {"repository": "registry.example.com/sie-placement", "tag": " "}}, "placementApplier.image"), + (None, {"apply": "true"}, 'placementApplier.apply must be a boolean (true or false), got "true"'), + (None, {"enabled": "true"}, 'placementApplier.enabled must be a boolean (true or false), got "true"'), + (None, {"plan": {}}, "placementApplier.plan must be a placement-plan document"), + (plan_without_lanes, {}, "placementApplier.plan.lanes must list at least one lane"), + ( + disable_h100, + {}, + 'placementApplier.plan.lanes[4] names the lane "batch|h100|default" (queuePool|machineProfile|bundle), ' + "which is not an enabled worker lane", + ), + ( + drop_l4_envelope, + {}, + 'placementApplier.plan.lanes[0] names workers.pools.l4.bundles["default"], which declares no envelope', + ), + ( + widen_l4_envelope, + {}, + "placementApplier.plan.lanes[0].envelope (floorMin 1, floorMax 2) differs from " + 'workers.pools.l4.bundles["default"].envelope (floorMin 1, floorMax 3)', + ), + (None, {"podAnnotations": {"checksum/plan": "x"}}, "must not override chart-owned annotation checksum/plan"), + ( + lambda layers: layers.insert(3, APPLIER_AUTH), + {}, + "placementApplier.enabled with gateway token auth (or a gateway auth mode known only at run time) needs " + "placementApplier.clientToken.existingSecretName", + ), + ( + lambda layers: layers.insert( + 3, {"gateway": {"auth": {"mode": "token", "tokenSecretName": "gateway-tokens"}}} + ), + {"apply": True, "clientToken": APPLIER_CLIENT_TOKEN}, + "placementApplier.apply=true with gateway token auth (or a gateway auth mode known only at run time) needs " + "gateway.auth.adminTokenSecretName", + ), + ( + lambda layers: layers.insert(3, APPLIER_AUTH), + {"clientToken": {"existingSecretName": "applier-client", "secretKey": " "}}, + "placementApplier.clientToken.secretKey is empty", + ), + ( + lambda layers: layers.insert( + 3, + { + "gateway": { + "extraEnv": [ + { + "name": "SIE_GATEWAY_MAX_POOLS", + "valueFrom": {"configMapKeyRef": {"name": "limits", "key": "pools"}}, + } + ] + } + }, + ), + {}, + "placementApplier.enabled needs gateway.extraEnv SIE_GATEWAY_MAX_POOLS as a literal value", + ), + ], +) +def test_an_unrealizable_placement_applier_fails_the_render( + tmp_path: Path, change: object, applier: dict, error: str +) -> None: + layers = applier_layers(**applier) + if callable(change): + change(layers) + result = render_placement(tmp_path, *layers) + assert result.returncode != 0 + assert error in result.stderr + + +def test_the_applier_client_token_is_kept_apart_from_the_admin_token(tmp_path: Path) -> None: + client = {"existingSecretName": "gateway-admin", "secretKey": "admin"} + result = render_placement(tmp_path, *applier_layers(APPLIER_AUTH, clientToken=client)) + assert result.returncode != 0 + assert ( + "The placement applier client token and the gateway admin token are the same Secret key (gateway-admin/admin)" + in result.stderr + ) + + +def envelope_values(settings: dict, *, autoscaling: bool = False) -> dict: + values = {"workers": {"pools": {"l4": {"enabled": True, "bundles": {"default": settings}}}}} + return {**values, **AUTOSCALING_VALUES} if autoscaling else values + + +ENVELOPE = 'workers.pools.l4.bundles["default"].envelope' + + +@pytest.mark.parametrize( + ("settings", "autoscaling"), + [ + ({"minReplicas": 0, "maxReplicas": 4, "envelope": {"floorMin": 0, "floorMax": 0}}, True), + ({"minReplicas": 0, "maxReplicas": 4, "envelope": {"floorMin": 0, "floorMax": 2}}, True), + ({"minReplicas": 0, "maxReplicas": 4, "envelope": {"floorMin": 0, "floorMax": 1}}, False), + ({"minReplicas": 0, "maxReplicas": 1, "envelope": {"floorMin": 0, "floorMax": 1}}, True), + ({"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": 4}}, True), + ], +) +def test_a_lane_envelope_inside_the_lane_bounds_renders(tmp_path: Path, settings: dict, autoscaling: bool) -> None: + result = render_workers(tmp_path, envelope_values(settings, autoscaling=autoscaling)) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + ("settings", "autoscaling", "error"), + [ + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 0, "floorMax": 2}}, + False, + f"{ENVELOPE}.floorMin must equal minReplicas (1), got 0", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": 5}}, + False, + f"{ENVELOPE}.floorMax must not exceed maxReplicas (4), got 5", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": 0}}, + False, + f"{ENVELOPE}.floorMax must be at least floorMin (1), got 0", + ), + ( + {"minReplicas": 0, "maxReplicas": 4, "envelope": {"floorMin": 0, "floorMax": 1}}, + True, + f"{ENVELOPE}.floorMax must be 0 or at least the lane's fallback floor (2), got 1", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": "2"}}, + False, + f"{ENVELOPE}.floorMax must be a non-negative integer, got 2", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": 1.5}}, + False, + f"{ENVELOPE}.floorMax must be a non-negative integer, got 1.5", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1}}, + False, + f"{ENVELOPE}.floorMax must be a non-negative integer, got nothing", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": {"floorMin": 1, "floorMax": 2, "bufferMin": 0}}, + False, + f"{ENVELOPE}.bufferMin is not an envelope field", + ), + ( + {"minReplicas": 1, "maxReplicas": 4, "envelope": [1, 2]}, + False, + f"{ENVELOPE} must be a map with floorMin and floorMax", + ), + ], +) +def test_a_lane_envelope_outside_the_lane_bounds_fails_the_render( + tmp_path: Path, settings: dict, autoscaling: bool, error: str +) -> None: + result = render_workers(tmp_path, envelope_values(settings, autoscaling=autoscaling)) + assert result.returncode != 0 + assert error in result.stderr + + +def test_readme_envelope_example_renders(tmp_path: Path) -> None: + readme = (ROOT / helm.CHART_DIR / "README.md").read_text(encoding="utf-8") + section = readme.split("### Lane envelopes and the placement applier\n", 1)[1].split("\n### ", 1)[0] + (example,) = re.findall(r"```yaml\n(.*?)```", section, flags=re.DOTALL) + values = yaml.safe_load(example) + assert values["workers"]["pools"]["l4"]["bundles"]["default"]["envelope"] + result = render_workers(tmp_path, {**values, **AUTOSCALING_VALUES}) + assert result.returncode == 0, result.stderr From f819b72b691afde492122380a46bbefb40c14c46 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 05:43:03 +0100 Subject: [PATCH 07/40] fix(placement): price the next cheaper option by planning again nextCheaper.saving is now the fall in the plan total found by planning again from the plan the run wrote, with each bound that blocks the option loosened to what the option measured, rounded up and never below 1. It is 0 when the total would not fall. The loosened bounds may admit a cheaper option still, and the saving counts it, as savingAtPermittedBound does for a come-alive lever. An identity that stays beside another resident of its current lane no longer claims the lane's floor as a nextCheaper saving: leaving the lane leaves that floor paid by the other residents. Levers and nextCheaper share one cached replan per loosened bound set. --- packages/sie_placement/README.md | 15 +++-- .../src/sie_placement/planner.py | 60 ++++++++++++------- .../fixtures/planner/expected-report.json | 2 +- packages/sie_placement/tests/test_planner.py | 57 +++++++++++++++++- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 72a4f4196..8f8322222 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -210,11 +210,16 @@ The report renders as a `placement-report` document: demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or `borrowed-boot-evidence`), the peak containers its demand needs, and - `nextCheaper`: the nearest cheaper option a service level blocks, its saving - and the bounds that block it. A kept identity's `blockedBy` names the bounds - its kept placement is known to miss: its receipt's warm latencies, a measured - come-alive over the bound, or a cold warmth under a bound of `0`, which needs - no measurement. + `nextCheaper`: the nearest cheaper option a service level blocks, the bounds + that block it, and `saving`, the fall in the plan total found by planning + again from this plan with each of those bounds loosened to what the option + measured, rounded up to a whole second or millisecond and never below 1. The + loosened bounds may admit a cheaper option still, and the saving counts it. + The saving is 0 when the total would not fall, as for an identity whose lane + keeps its floor for its other residents. A kept identity's `blockedBy` names + the bounds its kept placement is known to miss: its receipt's warm + latencies, a measured come-alive over the bound, or a cold warmth under a + bound of `0`, which needs no measurement. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 2ca4306bd..63c55ccb4 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -58,6 +58,12 @@ COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") +_LEVEL_FIELDS = { + "comeAliveP95S": "come_alive_p95_s", + "warmP99Ms": "warm_p99_ms", + "ttftP99Ms": "ttft_p99_ms", + "itlP99Ms": "itl_p99_ms", +} class PlannerInputError(ValueError): @@ -371,6 +377,7 @@ def __init__( self.current = {entry.identity: (entry, lanes[entry.lane]) for entry in current_plan.identities} self.refusals: list[Refusal] = [] self.refused_profiles: set[tuple[str, str]] = set() + self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} self.receipts = self._accept_receipts(receipts) self.boots = _boot_index(self._accept_boot_evidence(boot_evidence)) @@ -546,19 +553,25 @@ def replanned_total( current, self.parameters, ) - return planner.run(price_levers=False).report.monthly_cost + return planner.run(price_savings=False).report.monthly_cost - def loosened(self, identity: str, bound: int) -> ServiceLevels: + def loosened(self, identity: str, bounds: Mapping[str, int]) -> ServiceLevels: + fields = {_LEVEL_FIELDS[name]: bound for name, bound in bounds.items()} return replace( self.service_levels, identities=tuple( - replace(entry, level=replace(entry.level, come_alive_p95_s=bound)) - if entry.identity == identity - else entry + replace(entry, level=replace(entry.level, **fields)) if entry.identity == identity else entry for entry in self.service_levels.identities ), ) + def saving_when_loosened(self, identity: str, bounds: Mapping[str, int], written: Plan, total: float) -> float: + key = (identity, tuple(sorted(bounds.items()))) + if key not in self.savings: + replanned = self.replanned_total(written, service_levels=self.loosened(identity, bounds)) + self.savings[key] = _saving(total, replanned) + return self.savings[key] + def booted_within(self, identity: str, option: _Option, bound: int) -> tuple[BootEvidence, ...]: key = (identity, option.receipt.cell, option.receipt.container_shape, option.warmth) return tuple( @@ -583,14 +596,10 @@ def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: known = best.get(key) if known is None or option.rank < known.rank: best[key] = option - savings: dict[int, float] = {} levers = [] for option in best.values(): - permitted = max(1, math.ceil(option.come_alive_s)) - if permitted not in savings: - replanned = self.replanned_total(written, service_levels=self.loosened(choice.identity, permitted)) - savings[permitted] = _saving(total, replanned) - saving = savings[permitted] + permitted = _permitted(option.come_alive_s) + saving = self.saving_when_loosened(choice.identity, {"comeAliveP95S": permitted}, written, total) if not saving: continue under = 0.0 @@ -616,16 +625,19 @@ def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: ) return levers - @staticmethod - def next_cheaper(choice: _Choice) -> _Option | None: + def next_cheaper(self, choice: _Choice, written: Plan, total: float) -> Option | None: chosen = choice.chosen if chosen is None: return None cheaper = [option for option in choice.options if not option.feasible and option.cost < chosen.cost] if not cheaper: return None - nearest = max(option.cost for option in cheaper) - return min((option for option in cheaper if option.cost == nearest), key=lambda option: option.rank) + cost = max(option.cost for option in cheaper) + nearest = min((option for option in cheaper if option.cost == cost), key=lambda option: option.rank) + bounds = { + blocker.field: _permitted(blocker.measured) for blocker in nearest.blockers if blocker.measured is not None + } + return nearest.summary(saving=self.saving_when_loosened(choice.identity, bounds, written, total)) @staticmethod def lane_names(placed: Mapping[str, _Option], reserved: set[str]) -> dict[str, str]: @@ -706,7 +718,7 @@ def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) - return window.above_floor_hours(requests, rate, lane.envelope.floor_min) return window.cold_container_hours(requests, rate) - def run(self, *, price_levers: bool = True) -> PlannerResult: + def run(self, *, price_savings: bool = True) -> PlannerResult: choices = [self.choose(identity) for identity in sorted(self.levels)] placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} kept = { @@ -741,10 +753,14 @@ def run(self, *, price_levers: bool = True) -> PlannerResult: for choice in choices: requests.extend(self.receipt_requests(choice)) requests.extend(self.boot_requests(choice)) - if price_levers: + nearest = None + if price_savings: levers.extend(self.levers(choice, new_plan, total)) + nearest = self.next_cheaper(choice, new_plan, total) reports.append( - self._identity_report(choice, names, kept, costs.get(choice.identity), receipts.get(choice.identity)) + self._identity_report( + choice, names, kept, costs.get(choice.identity), receipts.get(choice.identity), nearest + ) ) identity_changes, lane_changes = _changes(self.current_plan, new_plan) report = Report( @@ -772,6 +788,7 @@ def _identity_report( kept: Mapping[str, tuple[PlanIdentity, PlanLane]], cost: float | None, receipt: Receipt | None, + next_cheaper: Option | None, ) -> IdentityReport: current = self.current.get(choice.identity) report = IdentityReport( @@ -795,7 +812,6 @@ def _identity_report( blocked_by=self.kept_blockers(choice.identity, entry, lane, receipt), ) return replace(report, unplaced_reason=reason) - nearest = self.next_cheaper(choice) rate = chosen.receipt.throughput.requests_per_second return replace( report, @@ -804,7 +820,7 @@ def _identity_report( monthly_cost=cost, chosen=chosen.summary(), peak_containers=max((_Window.needed(count, rate) for count in choice.requests), default=0), - next_cheaper=None if nearest is None else nearest.summary(saving=chosen.cost - nearest.cost), + next_cheaper=next_cheaper, ) def _pricing( @@ -848,6 +864,10 @@ def _saving(total: float, replanned: float) -> float: return saving if round(saving, 2) > 0 else 0.0 +def _permitted(measured: float) -> int: + return max(1, math.ceil(measured)) + + def _shared_chart_lanes(plan: Plan) -> tuple[SharedChartLane, ...]: grouped: defaultdict[tuple[str, str, str], list[str]] = defaultdict(list) for lane in plan.lanes: diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index e231c560c..5d4325782 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -69,7 +69,7 @@ "concurrency": 16, "warmth": "snapshot-cold", "monthlyCost": 569.4, - "saving": 189.8, + "saving": 0.0, "comeAliveS": 8.5, "comeAliveBasis": "boot-evidence", "blockedBy": [ diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index d1065bbd1..de33e77a4 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -254,6 +254,25 @@ def loosened(service_levels: ServiceLevels, identity: str, bound: int) -> Servic ) +LEVEL_FIELDS = { + "comeAliveP95S": "come_alive_p95_s", + "warmP99Ms": "warm_p99_ms", + "ttftP99Ms": "ttft_p99_ms", + "itlP99Ms": "itl_p99_ms", +} + + +def relaxed(service_levels: ServiceLevels, identity: str, blockers) -> ServiceLevels: + fields = {LEVEL_FIELDS[blocker.field]: max(1, math.ceil(blocker.measured)) for blocker in blockers} + return replace( + service_levels, + identities=tuple( + replace(entry, level=replace(entry.level, **fields)) if entry.identity == identity else entry + for entry in service_levels.identities + ), + ) + + def test_container_hourly_cost_prices_accelerators_cores_and_memory(): price = MachineProfilePrice( name="l4", gpu_product="NVIDIA-L4", accelerator_per_hour=2.0, core_per_hour=0.05, memory_gib_per_hour=0.01 @@ -496,6 +515,7 @@ def test_generation_cells_meet_both_the_first_token_and_the_per_token_bound(): Blocker(field="ttftP99Ms", bound=1500, measured=2000.0), Blocker(field="itlP99Ms", bound=80, measured=95.0), ) + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) def test_a_receipt_whose_realized_product_differs_from_its_profile_is_refused(): @@ -627,6 +647,23 @@ def test_the_next_cheaper_option_is_the_nearest_cheaper_one_and_names_its_bound( assert report.next_cheaper.blocked_by == (Blocker(field="warmP99Ms", bound=250, measured=300.0),) +@pytest.mark.parametrize(("others", "saving"), [((), MONTH), (("org/kept",), 0.0)]) +def test_a_next_cheaper_saving_is_what_loosening_its_bounds_saves_the_plan(others, saving): + service_levels = levels(*((name, ServiceLevel(come_alive_p95_s=0)) for name in (IDENTITY, *others))) + current = single_lane_plan(others=others) + result = run(service_levels, [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=current) + report = result.report.identity(IDENTITY) + assert (report.lane, report.chosen.warmth) == ("shared", Warmth.PINNED) + assert (report.next_cheaper.warmth, report.next_cheaper.monthly_cost) == (Warmth.COLD_WITH_BOUND, 0.0) + assert report.chosen.monthly_cost - report.next_cheaper.monthly_cost == pytest.approx(MONTH) + assert report.next_cheaper.saving == pytest.approx(saving) + again = run( + loosened(service_levels, IDENTITY, 30), [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=result.plan + ) + assert again.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + assert result.report.monthly_cost - again.report.monthly_cost == pytest.approx(saving) + + def test_a_placement_with_nothing_cheaper_names_no_next_option(): result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY) assert result.report.identity(IDENTITY).next_cheaper is None @@ -1157,12 +1194,14 @@ def test_an_estate_lever_saves_what_planning_again_at_its_permitted_bound_saves( assert lever.saving_at_permitted_bound > 0 -def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever(): +def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever_and_no_saving(): inputs = estate() result = plan(**inputs) dense = result.report.identity("BAAI/bge-m3") assert (dense.lane, dense.chosen.warmth) == ("default-l4-default", Warmth.PINNED) assert dense.next_cheaper.come_alive_s == 8.5 + assert dense.chosen.monthly_cost > dense.next_cheaper.monthly_cost + assert dense.next_cheaper.saving == 0 assert not [lever for lever in result.report.come_alive_levers if lever.identity == "BAAI/bge-m3"] for bound in (9, 95): service_levels = loosened(inputs["service_levels"], "BAAI/bge-m3", bound) @@ -1171,6 +1210,18 @@ def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever(): assert again.report.monthly_cost > result.report.monthly_cost +def test_an_estate_next_cheaper_saving_is_what_planning_again_with_its_bounds_loosened_saves(): + inputs = estate() + result = plan(**inputs) + offered = [entry for entry in result.report.identities if entry.next_cheaper is not None] + assert [entry.identity for entry in offered] == ["BAAI/bge-m3", "Qwen/Qwen3-8B", "Qwen/Qwen3-Embedding-4B"] + for entry in offered: + service_levels = relaxed(inputs["service_levels"], entry.identity, entry.next_cheaper.blocked_by) + again = plan(**{**inputs, "service_levels": service_levels, "current_plan": result.plan}) + fall = result.report.monthly_cost - again.report.monthly_cost + assert entry.next_cheaper.saving == pytest.approx(max(0.0, fall), abs=0.01) + + @pytest.mark.parametrize("seed", range(25)) def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed): rng = random.Random(seed) # noqa: S311 -- reproducible test data @@ -1243,6 +1294,10 @@ def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed if entry.next_cheaper is not None: assert entry.next_cheaper.blocked_by assert entry.next_cheaper.monthly_cost < chosen.monthly_cost + looser = relaxed(service_levels, entry.identity, entry.next_cheaper.blocked_by) + again = run(looser, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) + fall = result.report.monthly_cost - again.report.monthly_cost + assert entry.next_cheaper.saving == pytest.approx(max(0.0, fall), abs=0.01) costs = {entry.identity: entry.chosen.monthly_cost for entry in result.report.identities if entry.chosen} for lever in result.report.come_alive_levers: assert lever.monthly_cost < costs[lever.identity] From 2592c21f813bbc3948927abf50b9972d7ad49628 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 06:09:14 +0100 Subject: [PATCH 08/40] fix(helm): suggest only single-token Secrets to the placement applier The applier presents its client token file whole, so a gateway token Secret that lists several comma-separated tokens fails every pool read. The render error now suggests the gateway token Secret only when the chart generates it, which holds one token, and states that the key must hold one token. The chart README states that the applier reads both token files at start and needs a restart after a gateway client or admin token rotation, names the applier's client token in the gateway token rotation steps, and states that the Helm values emitted from a plan carry no lane envelopes, so they are copied from the plan. New render tests cover a gateway auth mode known only at run time, apply mode without gateway auth, the bundled Prometheus address, and the --max-pools flag. --- deploy/helm/sie-cluster/README.md | 24 +++- .../helm/sie-cluster/templates/_helpers.tpl | 4 +- tools/ci/tests/test_helm_render.py | 108 ++++++++++++++++-- 3 files changed, 125 insertions(+), 11 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 36b46d307..013531dec 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -561,10 +561,16 @@ The render fails unless: an enabled worker lane by its `queuePool`, `cell.machineProfile` and `bundle`, compared as the chart canonicalizes lane tokens, and that worker lane declares an `envelope` equal to the plan lane's. A plan therefore cannot - move a floor outside a range that the chart values bound. + move a floor outside a range that the chart values bound. The values overlay + that `helm_values` in `sie-placement` emits from a plan sets each lane's + `minReplicas` and `maxReplicas` but no `envelope`, so copy each plan lane's + `envelope.floorMin` and `envelope.floorMax` into the worker lane it names. - With gateway token auth, or a gateway auth mode known only at run time, `placementApplier.clientToken.existingSecretName` names a Secret key that holds one gateway client token, which the applier presents on pool reads. + The applier presents the whole value, so the key must not list several + comma-separated tokens, as an operator-managed `gateway.auth.tokenSecretName` + does during a [rotation](#gateway-token). - With `placementApplier.apply: true` and gateway token auth, or a mode known only at run time, `gateway.auth.adminTokenSecretName` is set. - `gateway.extraEnv` sets `SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT` and @@ -599,6 +605,18 @@ floors, not credentials. Anyone who can create pods in the namespace with that Secret can still read it, so the Secret read limits under [Who holds which token](#who-holds-which-token) apply. +The applier reads both token files once, when it starts. After you rotate the +gateway client token or the gateway admin token, restart it: + +```bash +kubectl rollout restart deployment/-placement-applier -n +``` + +Until it restarts, the gateway refuses its pool reads or writes and the pod +keeps running: each tick's audit rows record the failures, and in apply mode +each carrier lapses 1200 seconds after its last write and the lane returns to +its chart floor. + The Deployment runs one replica with the `Recreate` strategy, so at most one applier writes carriers, also during a rollout. Its pod runs as user and group 65532 with `runAsNonRoot`, the `RuntimeDefault` seccomp profile, a read-only @@ -1848,7 +1866,9 @@ without a cutover, use an operator-managed Secret: `helm upgrade` (with the Secret's name in `gateway.auth.tokenSecretName`) and restart the gateway and the worker StatefulSets. The gateway accepts both tokens, and the worker sidecars present the first one. -2. Move every client, including the MCP edge's key, to the new token. +2. Move every client, including the MCP edge's key and the placement + applier's `placementApplier.clientToken` key, to the new token, and restart + the placement applier, which reads its token at start. 3. Remove the old token from the Secret and restart the gateway. To leave a generated token, copy it into your own Secret as the old token in diff --git a/deploy/helm/sie-cluster/templates/_helpers.tpl b/deploy/helm/sie-cluster/templates/_helpers.tpl index 40195b521..f67f372da 100644 --- a/deploy/helm/sie-cluster/templates/_helpers.tpl +++ b/deploy/helm/sie-cluster/templates/_helpers.tpl @@ -1092,10 +1092,12 @@ lanes and their envelopes. {{- $mayAuthenticate := or (include "sie-cluster.gateway.authenticates" $root) (not $auth.known) -}} {{- if and $mayAuthenticate (not $clientToken.existingSecretName) -}} {{- $example := "" -}} +{{- if not (dig "auth" "tokenSecretName" "" ($root.Values.gateway | default dict)) -}} {{- with include "sie-cluster.gateway.tokenSecretName" $root -}} {{- $example = printf ", for example placementApplier.clientToken.existingSecretName=%s and placementApplier.clientToken.secretKey=%s" . (include "sie-cluster.gateway.tokenSecretKey" $root) -}} {{- end -}} -{{- fail (printf "placementApplier.enabled with gateway token auth (or a gateway auth mode known only at run time) needs placementApplier.clientToken.existingSecretName: the applier reads pools from the gateway, which refuses requests without a token. Point it at a Secret key that holds one gateway client token%s." $example) -}} +{{- end -}} +{{- fail (printf "placementApplier.enabled with gateway token auth (or a gateway auth mode known only at run time) needs placementApplier.clientToken.existingSecretName: the applier reads pools from the gateway, which refuses requests without a token. Point it at a Secret key that holds one gateway client token%s. The applier presents the whole value, so the key must not list several comma-separated tokens." $example) -}} {{- end -}} {{- if and (include "sie-cluster.placementApplier.apply" $root) $mayAuthenticate (not (dig "auth" "adminTokenSecretName" "" ($root.Values.gateway | default dict))) -}} {{- fail "placementApplier.apply=true with gateway token auth (or a gateway auth mode known only at run time) needs gateway.auth.adminTokenSecretName: the gateway refuses pool writes without its admin token, which the applier reads from that Secret." -}} diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 5a667bac3..d4267cb56 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4239,6 +4239,10 @@ def test_placement_values_at_the_lane_limit_render( } } APPLIER_CLIENT_TOKEN = {"existingSecretName": "applier-client", "secretKey": "token"} +RUNTIME_AUTH_MODE = [ + {"name": "SIE_AUTH_MODE", "valueFrom": {"configMapKeyRef": {"name": "gateway-auth", "key": "mode"}}} +] +APPLIER_RUNTIME_AUTH = {"gateway": {**APPLIER_AUTH["gateway"], "extraEnv": RUNTIME_AUTH_MODE}} SECRET_FILE_MODE = 0o440 @@ -4355,8 +4359,9 @@ def test_the_placement_applier_runs_in_shadow_mode_without_credentials(tmp_path: @pytest.mark.parametrize("apply", [False, True]) -def test_the_placement_applier_mounts_the_admin_token_only_to_write(tmp_path: Path, apply: bool) -> None: - docs = placement_documents(tmp_path, applier_layers(APPLIER_AUTH, apply=apply, clientToken=APPLIER_CLIENT_TOKEN)) +@pytest.mark.parametrize("auth", [APPLIER_AUTH, APPLIER_RUNTIME_AUTH], ids=["token", "run-time"]) +def test_the_placement_applier_mounts_the_admin_token_only_to_write(tmp_path: Path, auth: dict, apply: bool) -> None: + docs = placement_documents(tmp_path, applier_layers(auth, apply=apply, clientToken=APPLIER_CLIENT_TOKEN)) spec = applier_pod(docs)["spec"] (container,) = spec["containers"] client = ["--api-key-file", "/etc/sie-placement/client-token/token"] @@ -4387,14 +4392,66 @@ def test_the_placement_applier_mounts_the_admin_token_only_to_write(tmp_path: Pa } -def test_the_placement_applier_uses_the_gateways_literal_pool_limits(tmp_path: Path) -> None: - extra_env = [ - {"name": "SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT", "value": " 6 "}, - {"name": "SIE_GATEWAY_MAX_POOLS", "value": "many"}, - ] +def test_the_placement_applier_writes_without_a_token_when_the_gateway_authenticates_nothing( + tmp_path: Path, +) -> None: + spec = applier_pod(placement_documents(tmp_path, applier_layers(apply=True)))["spec"] + (container,) = spec["containers"] + assert container["args"] == [*APPLIER_ARGS, "--apply"] + assert [mount["name"] for mount in container["volumeMounts"]] == ["plan"] + assert [volume["name"] for volume in spec["volumes"]] == ["plan"] + + +@pytest.mark.parametrize( + ("dependencies", "address"), + [ + ({}, "http://prometheus-operated.monitoring.svc.cluster.local:9090"), + ( + {"kube-prometheus-stack": {"install": True}}, + "http://sie-kube-prometheus-stack-prometheus.sie.svc.cluster.local:9090", + ), + ], +) +def test_the_placement_applier_queries_the_prometheus_that_keda_queries( + tmp_path: Path, dependencies: dict, address: str +) -> None: + docs = placement_documents(tmp_path, applier_layers(dependencies)) + (container,) = applier_pod(docs)["spec"]["containers"] + args = container["args"] + assert args[args.index("--prometheus-url") + 1] == address + assert { + trigger["metadata"]["serverAddress"] + for scaled_object in worker_scaled_objects(docs) + for trigger in scaled_object["spec"]["triggers"] + } == {address} + + +@pytest.mark.parametrize( + ("extra_env", "limits"), + [ + ( + [ + {"name": "SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT", "value": " 6 "}, + {"name": "SIE_GATEWAY_MAX_POOLS", "value": "many"}, + ], + ["--max-pool-floor", "6"], + ), + ([{"name": "SIE_GATEWAY_MAX_POOLS", "value": "32"}], ["--max-pools", "32"]), + ( + [ + {"name": "SIE_GATEWAY_MAX_POOLS", "value": "32"}, + {"name": "SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT", "value": "3"}, + ], + ["--max-pool-floor", "3", "--max-pools", "32"], + ), + ], +) +def test_the_placement_applier_uses_the_gateways_literal_pool_limits( + tmp_path: Path, extra_env: list[dict], limits: list[str] +) -> None: docs = placement_documents(tmp_path, applier_layers({"gateway": {"extraEnv": extra_env}})) (container,) = applier_pod(docs)["spec"]["containers"] - assert container["args"] == [*APPLIER_ARGS, "--max-pool-floor", "6"] + assert container["args"] == [*APPLIER_ARGS, *limits] def drop_l4_envelope(layers: list[dict]) -> None: @@ -4458,6 +4515,20 @@ def plan_without_lanes(layers: list[dict]) -> None: "placementApplier.apply=true with gateway token auth (or a gateway auth mode known only at run time) needs " "gateway.auth.adminTokenSecretName", ), + ( + lambda layers: layers.insert(3, APPLIER_RUNTIME_AUTH), + {}, + "placementApplier.enabled with gateway token auth (or a gateway auth mode known only at run time) needs " + "placementApplier.clientToken.existingSecretName", + ), + ( + lambda layers: layers.insert( + 3, {"gateway": {"auth": {"tokenSecretName": "gateway-tokens"}, "extraEnv": RUNTIME_AUTH_MODE}} + ), + {"apply": True, "clientToken": APPLIER_CLIENT_TOKEN}, + "placementApplier.apply=true with gateway token auth (or a gateway auth mode known only at run time) needs " + "gateway.auth.adminTokenSecretName", + ), ( lambda layers: layers.insert(3, APPLIER_AUTH), {"clientToken": {"existingSecretName": "applier-client", "secretKey": " "}}, @@ -4493,6 +4564,27 @@ def test_an_unrealizable_placement_applier_fails_the_render( assert error in result.stderr +@pytest.mark.parametrize( + ("auth", "example"), + [({"mode": "token", "generateToken": True}, True), ({"mode": "token", "tokenSecretName": "gateway-tokens"}, False)], + ids=["generated", "operator-managed"], +) +def test_the_applier_client_token_error_suggests_only_a_single_token_secret( + tmp_path: Path, auth: dict, example: bool +) -> None: + result = render_placement(tmp_path, *applier_layers({"gateway": {"auth": auth}})) + assert result.returncode != 0 + generated = ( + "for example placementApplier.clientToken.existingSecretName=sie-sie-cluster-gateway-token and " + "placementApplier.clientToken.secretKey=SIE_AUTH_TOKEN" + ) + assert (generated in result.stderr) is example + assert ("for example" in result.stderr) is example + assert "The applier presents the whole value, so the key must not list several comma-separated tokens" in ( + result.stderr + ) + + def test_the_applier_client_token_is_kept_apart_from_the_admin_token(tmp_path: Path) -> None: client = {"existingSecretName": "gateway-admin", "secretKey": "admin"} result = render_placement(tmp_path, *applier_layers(APPLIER_AUTH, clientToken=client)) From 884584bb64c20da9ad6e54dafdc004ce8120d7f9 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 13:23:52 +0100 Subject: [PATCH 09/40] feat(placement): keep unmeasured current placements and select lazy residents A first plan over a running deployment should change nothing it has no measured reason to change. Four selection rules make that hold: - An identity whose current placement no option matches, because its lane has no receipt at its cell, container shape and concurrency, a cold lane has no successful boot, or a lazy resident has no load, keeps that placement unless it is known to miss a bound. It leaves only for an unblocked option at least a cent cheaper than the placement, priced at the lane's fastest receipt or, without one, at its floor alone. What the placement lacks is a measurement request at its lane, with the lane's name and concurrency. An identity that stays keeps its lane as it is, envelope included, capped by the reviewed ceiling. - An identity that declares no warm bound has no bound to choose a class by: cells outside its current class are blocked by the undeclared bound, which the report lists, and an identity without a current lane is not placed. - lazy-resident is selectable on a warm lane of the current plan, on the identity's queue pool and bundle, that a never-cold identity pins, when the identity has a receipt at that lane and a load onto its cell and container shape. Its come-alive is the load and its cost the demand above the host's floor. A missing load is a measurement request on that lane. - A warmth's come-alive is the 95th percentile by nearest rank of its successful boots from 20 boots, and their slowest below that. Kept identities now report the nearest cheaper option and price their measurement requests against the current placement, and come-alive levers include lazy-resident. --- packages/sie_placement/README.md | 122 +++-- .../src/sie_placement/planner.py | 289 ++++++++-- .../sie_placement/src/sie_placement/report.py | 12 +- .../fixtures/planner/expected-report.json | 68 +-- packages/sie_placement/tests/test_planner.py | 508 +++++++++++++++++- 5 files changed, 870 insertions(+), 129 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 8f8322222..bc24b2f6c 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -121,9 +121,11 @@ print(render(result.plan.to_dict()), render(result.report.to_dict())) `plan` is a pure function of its documents. The same inputs render to the same bytes in whatever order the receipts, boot evidence, identities and demand -series arrive. An identity whose choice is its current placement keeps its -current lane and that lane's name, so a run whose `current_plan` already holds -every choice, such as its own previous plan, reports no change. +series arrive. An identity that stays on its current placement keeps its +current lane as it is, so a run whose `current_plan` already holds every +choice, such as its own previous plan, reports no change. The one exception is +a lane that the previous run created for an identity whose `comeAliveP95S` is +`0`: it hosts lazy residents only from the run after it. | Parameter | Meaning | | --- | --- | @@ -145,27 +147,54 @@ document: that profile, and a receipt whose latency cannot answer the identity's service level. The report lists every refusal. 2. Every other receipt is a cell. A cell whose p99 misses `warmP99Ms`, or - `ttftP99Ms` or `itlP99Ms`, is blocked. + `ttftP99Ms` or `itlP99Ms`, is blocked. An identity that declares none of + them has no bound to choose a class by: every cell outside its current + class, the cell of its current lane, is blocked by the undeclared bound, and + an identity without a current lane has no unblocked cell. 3. A cell offers `pinned`, and `cold-with-bound` or `snapshot-cold` where boot - evidence for the identity, cell and container shape exists. The slowest - successful boot stands for the come-alive percentile, a boot that did not - succeed withdraws the warmth, and the environment's own evidence replaces - borrowed evidence. A `comeAliveP95S` of `0` admits `pinned` only. -4. The cheapest unblocked option wins. Ties go to the current placement, + evidence for the identity, cell and container shape exists. Below 20 + successful boots the slowest stands for the come-alive percentile; from 20, + their 95th percentile by nearest rank does, so one slow boot stops deciding + once enough boots exist. A boot that did not succeed withdraws the warmth, + and the environment's own evidence replaces borrowed evidence. A + `comeAliveP95S` of `0` admits `pinned` only. +4. A cell also offers `lazy-resident` on each host lane with its cell, + container shape and concurrency. A host lane is a lane of the current plan + with a `floorMin` of at least 1, on the parameters' queue pool and the + identity's bundle, on which another identity whose `comeAliveP95S` is `0` is + pinned. The identity's own current lane is a host only when it is a lazy + resident there, since its floor is paid either way. + The come-alive is the identity's `lazy-resident` boot evidence, its measured + load onto that cell and container shape, under the same statistic. The + option costs what the identity's demand needs above the host lane's floor. +5. The cheapest unblocked option wins. Ties go to the current placement, then to the faster come-alive. The current placement is the identity's current lane when that lane has the option's cell, container shape and concurrency, the identity's warmth is the option's, and the lane's `queuePool` and `bundle` are the ones the parameters -give the identity. An identity whose choice is its current placement stays on -that lane under its name, beside whoever else the lane hosts, and the lane -takes the ceiling and envelope of the choice. Every other placed identity gets -a new lane of its own, named from the identity and its machine profile, with a -digest of the identity added when that name is taken. The planner does not -select `lazy-resident`, `bridged` or `scheduled-warm`, and does not move an -identity onto another identity's lane. An identity with no unblocked option -keeps its current lane, capped by the reviewed ceiling, or stays out of the -plan when it has none. +give the identity; a `lazy-resident` option must also be hosted by that lane. +A current placement that no option matches lacks evidence: a receipt at its +lane's cell, container shape and concurrency, a successful boot for a cold +warmth, or a load for a lazy resident. Unless it is known to miss a bound, +through a receipt at the lane that misses a warm bound, a measured come-alive +over the bound, or a cold or `lazy-resident` warmth under a bound of `0`, the +identity keeps that placement and leaves it only for an unblocked option at +least a cent cheaper than the placement. The placement is priced as an option +of its warmth at the lane's fastest receipt, or without one at its floor alone: +one container all month for `pinned`, nothing beyond the host lane's floor for +`lazy-resident`, and the burst tails for a cold warmth. What the placement +lacks becomes a measurement request. + +An identity that stays on its current placement keeps that lane exactly as it +is, beside whoever else the lane hosts: its cell, container shape, +concurrency and envelope, capped by the reviewed ceiling. A lazy resident +joins its host lane the same way. Every other placed identity gets a new lane +of its own, named from the identity and its machine profile, with a digest of +the identity added when that name is taken. The planner does not select +`bridged` or `scheduled-warm`. An identity with no unblocked option keeps its +current lane, capped by the reviewed ceiling, or stays out of the plan when it +has none. ### Pricing @@ -191,12 +220,13 @@ cell, container shape and concurrency, under that lane's own ceiling. ### Envelope -`floorMin` is 1 for `pinned` and 0 otherwise. No source measures queue wait -yet, so `floorMax` is `floorMin` plus `buffer_containers`, capped by the -ceiling. A floor raised on a zero-floor lane realizes as at least +A new lane's `floorMin` is 1 for `pinned` and 0 otherwise. No source measures +queue wait yet, so `floorMax` is `floorMin` plus `buffer_containers`, capped by +the ceiling. A floor raised on a zero-floor lane realizes as at least `fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least `fallback_floor` capped by the ceiling. The envelope carries the floor range -only. +only. A lane carried from the current plan keeps its envelope, capped by the +ceiling. ### Report @@ -209,17 +239,24 @@ The report renders as a `placement-report` document: effective service level and demand, its lane and `monthlyCost`, what its demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or - `borrowed-boot-evidence`), the peak containers its demand needs, and - `nextCheaper`: the nearest cheaper option a service level blocks, the bounds - that block it, and `saving`, the fall in the plan total found by planning - again from this plan with each of those bounds loosened to what the option - measured, rounded up to a whole second or millisecond and never below 1. The - loosened bounds may admit a cheaper option still, and the saving counts it. - The saving is 0 when the total would not fall, as for an identity whose lane - keeps its floor for its other residents. A kept identity's `blockedBy` names - the bounds its kept placement is known to miss: its receipt's warm - latencies, a measured come-alive over the bound, or a cold warmth under a - bound of `0`, which needs no measurement. + `borrowed-boot-evidence`) and, for a lazy resident, its host `lane`, the peak + containers its demand needs, and `nextCheaper`: the nearest option cheaper + than the choice, or than a kept identity's current placement, that a + service level blocks, the bounds that block it, and `saving`, the fall in + the plan total found by planning again from this plan with each of those + bounds loosened to what the option measured, rounded up to a whole second or + millisecond and never below 1. A bound the identity does not declare blocks + with no `bound`, and loosening it declares it. The loosened bounds may admit + a cheaper option still, and the saving counts it. The saving is 0 when the + total would not fall, as for an identity whose lane keeps its floor for its + other residents. An identity without a placement names its + `unplacedReason`: `no-receipt`, `warm-bound-missed`, + `warm-bound-undeclared` when no cell is on its current class, or + `current-unmeasured` when it keeps a current placement that lacks evidence + and no unblocked option is cheaper. A kept identity's `blockedBy` names the + bounds its kept placement is known to miss: its receipt's warm latencies, a + measured come-alive over the bound, or a cold or `lazy-resident` warmth under + a bound of `0`, which needs no measurement. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, @@ -231,11 +268,18 @@ The report renders as a `placement-report` document: could unlock. A profile without a receipt is bounded by price alone: one accelerator at its rate, kept warm or, where the come-alive bound allows, paying burst tails only. A cell whose receipt has no successful boot asks for - boot evidence and is bounded by its cold cost. A request that cannot save is - dropped, and the requests of an identity without a placement carry no bound - and rank first. -- `comeAliveLevers`: cold warmths whose measured come-alive misses the bound - on a cell that meets the warm bounds for less than the choice, listed only + boot evidence and is bounded by its cold cost, and a host lane without the + identity's load asks for `lazy-resident` boot evidence on it, bounded by the + demand above the host's floor. Savings are measured against the choice, or + against a kept identity's current placement. A request that cannot save is + dropped, and the requests of an identity without a placement or a current + one carry no bound and rank first. The evidence a kept current placement + lacks is requested at its lane, named in `lane` with its `concurrency`, and + also ranks first without a bound; a profile it asks for there is not asked + for again. +- `comeAliveLevers`: cold and `lazy-resident` warmths whose measured + come-alive misses the bound on a cell that meets the warm bounds for less + than the choice, or than a kept identity's current placement, listed only when loosening the identity's bound to `permittedBoundS`, the bound the measured come-alive would permit, lowers the plan total. Both savings are found by planning again from this plan and taking the fall in its total: @@ -247,6 +291,8 @@ The report renders as a `placement-report` document: then by the saving at the permitted bound. - `queueWaitUnattained`: every identity that declares `queueWaitP99Ms`. No plan claims that bound as met. +- `warmBoundUndeclared`: every identity that has a cell and declares neither + `warmP99Ms` nor `ttftP99Ms` and `itlP99Ms`, so keeps its current class. - `refusals`, and `changes`: the identities and lanes that differ from `current_plan`. diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 63c55ccb4..88ed83bcb 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -31,6 +31,7 @@ UnplacedReason, ) from sie_placement.schema import ( + BOOT_WARMTHS, WARM_FLOOR_WARMTHS, BootEvidence, BootOutcome, @@ -55,6 +56,8 @@ SECONDS_PER_HOUR = 3600 HOURS_PER_MONTH = 730.0 PINNED_FLOOR = 1 +BOOT_PERCENTILE = 95 +BOOT_PERCENTILE_MIN_ROWS = 20 COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") @@ -209,14 +212,15 @@ class _Option: basis: ComeAliveBasis blockers: tuple[Blocker, ...] current: bool + host: str | None = None @property def feasible(self) -> bool: return not self.blockers @property - def rank(self) -> tuple[float, bool, float, str, str]: - return (self.cost, not self.current, self.come_alive_s, self.order, self.warmth.value) + def rank(self) -> tuple[float, bool, float, str, str, str]: + return (self.cost, not self.current, self.come_alive_s, self.order, self.warmth.value, self.host or "") def summary(self, saving: float | None = None) -> Option: return Option( @@ -230,6 +234,7 @@ def summary(self, saving: float | None = None) -> Option: receipt=self.receipt.source, blocked_by=self.blockers, saving=saving, + lane=self.host, ) @@ -242,6 +247,22 @@ class _Choice: chosen: _Option | None boot_requests: tuple[MeasurementRequest, ...] has_valid_receipt: bool + held: float | None = None + gaps: tuple[MeasurementRequest, ...] = () + + @property + def baseline(self) -> float | None: + return self.held if self.chosen is None else self.chosen.cost + + @property + def unplaced_reason(self) -> UnplacedReason: + if any(option.feasible for option in self.options): + return UnplacedReason.CURRENT_UNMEASURED + if not self.has_valid_receipt: + return UnplacedReason.NO_RECEIPT + if not _declares_warm_bound(self.level): + return UnplacedReason.WARM_BOUND_UNDECLARED + return UnplacedReason.WARM_BOUND_MISSED def _canonical(document: Mapping[str, object]) -> str: @@ -292,6 +313,19 @@ def _warm_blockers(level: ServiceLevel, receipt: Receipt) -> tuple[Blocker, ...] ) +def _declares_warm_bound(level: ServiceLevel) -> bool: + return level.warm_p99_ms is not None or level.is_generation + + +def _undeclared_blockers(level: ServiceLevel, receipt: Receipt) -> tuple[Blocker, ...]: + if _declares_warm_bound(level): + return () + measured = (("warmP99Ms", receipt.latency), ("ttftP99Ms", receipt.ttft), ("itlP99Ms", receipt.itl)) + return tuple( + Blocker(field=name, bound=None, measured=values.p99_ms) for name, values in measured if values is not None + ) + + def _come_alive_blocker(level: ServiceLevel, seconds: float) -> Blocker | None: bound = level.come_alive_p95_s if bound is None or (bound > 0 and seconds <= bound): @@ -299,6 +333,13 @@ def _come_alive_blocker(level: ServiceLevel, seconds: float) -> Blocker | None: return Blocker(field="comeAliveP95S", bound=bound, measured=seconds) +def _come_alive(seconds: Iterable[float]) -> float: + ordered = sorted(float(value) for value in seconds) + if len(ordered) < BOOT_PERCENTILE_MIN_ROWS: + return ordered[-1] + return ordered[(len(ordered) * BOOT_PERCENTILE + 99) // 100 - 1] + + def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: grouped: defaultdict[_BootKey, list[BootEvidence]] = defaultdict(list) for record in records: @@ -310,7 +351,7 @@ def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: if any(record.outcome is not BootOutcome.SUCCEEDED for record in used): index[key] = _Boot(seconds=None, borrowed=not own) else: - index[key] = _Boot(seconds=float(max(record.boot_seconds for record in used)), borrowed=not own) + index[key] = _Boot(seconds=_come_alive(record.boot_seconds for record in used), borrowed=not own) return index @@ -372,9 +413,15 @@ def __init__( self.requests = _hourly_requests(demand) self.current_plan = current_plan self.current: dict[str, tuple[PlanIdentity, PlanLane]] = {} + self.current_lanes: dict[str, PlanLane] = {} + self.residents: defaultdict[str, list[PlanIdentity]] = defaultdict(list) if current_plan is not None: - lanes = {lane.name: lane for lane in current_plan.lanes} - self.current = {entry.identity: (entry, lanes[entry.lane]) for entry in current_plan.identities} + self.current_lanes = {lane.name: lane for lane in current_plan.lanes} + self.current = { + entry.identity: (entry, self.current_lanes[entry.lane]) for entry in current_plan.identities + } + for entry in current_plan.identities: + self.residents[entry.lane].append(entry) self.refusals: list[Refusal] = [] self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} @@ -422,18 +469,118 @@ def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: and _runs_on(receipt, lane) ) + def current_class(self, identity: str) -> Cell | None: + current = self.current.get(identity) + return None if current is None else current[1].cell + + def keepable(self, identity: str) -> tuple[PlanIdentity, PlanLane] | None: + current = self.current.get(identity) + if current is None: + return None + lane = current[1] + if lane.queue_pool != self.parameters.queue_pool or lane.bundle != self.parameters.bundle_of(identity): + return None + return current + + def anchors(self, entry: PlanIdentity) -> bool: + level = self.levels.get(entry.identity) + return entry.warmth is Warmth.PINNED and level is not None and level.come_alive_p95_s == 0 + + def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: + current = self.current.get(identity) + home = None if current is None else current[1].name + lazy = current is not None and current[0].warmth is Warmth.LAZY_RESIDENT + bundle = self.parameters.bundle_of(identity) + return tuple( + lane + for name, lane in sorted(self.current_lanes.items()) + if lane.envelope.floor_min >= PINNED_FLOOR + and lane.queue_pool == self.parameters.queue_pool + and lane.bundle == bundle + and ( + lazy + if name == home + else any(self.anchors(entry) for entry in self.residents[name] if entry.identity != identity) + ) + ) + + def resident_hours(self, lane: PlanLane, requests: Sequence[int], rate: float) -> float: + capped = self.within_ceiling(lane) + window = replace(self.window, ceiling=capped.max_containers) + return window.above_floor_hours(requests, rate, capped.envelope.floor_min) + + def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> float | None: + if _refusal_reason(lane.cell, self.prices) is not None: + return None + hourly = container_hourly_cost(self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier) + requests = self.requests.get(identity, ()) + rate = max( + ( + receipt.throughput.requests_per_second + for _, receipt in self.receipts.get(identity, []) + if _runs_on(receipt, lane) + ), + default=None, + ) + if entry.warmth is Warmth.PINNED: + hours = ( + PINNED_FLOOR * self.parameters.hours_per_month + if rate is None + else self.window.warm_container_hours(requests, rate) + ) + elif entry.warmth is Warmth.LAZY_RESIDENT: + hours = 0.0 if rate is None else self.resident_hours(lane, requests, rate) + else: + hours = self.window.cold_container_hours(requests, rate) + return hourly * hours + + def current_gaps( + self, identity: str, entry: PlanIdentity, lane: PlanLane, cost: float | None + ) -> tuple[MeasurementRequest, ...]: + if cost is None: + return () + missing: list[tuple[Measurement, RequestReason]] = [] + if self.slowest_receipt(identity, lane) is None: + refused = (identity, lane.cell.machine_profile) in self.refused_profiles + missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) + if entry.warmth in BOOT_WARMTHS: + boot = self.boots.get((identity, lane.cell, lane.container_shape, entry.warmth)) + if boot is None or boot.seconds is None: + reason = RequestReason.UNMEASURED if boot is None else RequestReason.FAILED + missing.append((Measurement.BOOT_EVIDENCE, reason)) + return tuple( + MeasurementRequest( + identity=identity, + cell=lane.cell, + container_shape=lane.container_shape, + concurrency=lane.concurrency, + lane=lane.name, + measure=(measurement,), + warmth=entry.warmth, + reason=reason, + cost_bound=cost, + saving_bound=None, + ) + for measurement, reason in missing + ) + def choose(self, identity: str) -> _Choice: level = self.levels[identity] requests = self.requests.get(identity, ()) options: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} + load_requests: dict[str, MeasurementRequest] = {} receipts = self.receipts.get(identity, []) + current_class = self.current_class(identity) + hosts = self.lazy_hosts(identity) for order, receipt in receipts: hourly = container_hourly_cost( self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier ) rate = receipt.throughput.requests_per_second warm = _warm_blockers(level, receipt) + if receipt.cell != current_class: + warm = (*warm, *_undeclared_blockers(level, receipt)) options.append( _Option( receipt=receipt, @@ -478,20 +625,82 @@ def choose(self, identity: str) -> _Choice: current=self.is_current(identity, receipt, warmth), ) ) + for host in hosts: + if not _runs_on(receipt, host): + continue + lazy_cost = hourly * self.resident_hours(host, requests, rate) + load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) + if load is None or load.seconds is None: + known = load_requests.get(host.name) + if not warm and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): + load_requests[host.name] = MeasurementRequest( + identity=identity, + cell=host.cell, + container_shape=host.container_shape, + concurrency=host.concurrency, + lane=host.name, + measure=(Measurement.BOOT_EVIDENCE,), + warmth=Warmth.LAZY_RESIDENT, + reason=RequestReason.UNMEASURED if load is None else RequestReason.FAILED, + cost_bound=lazy_cost, + saving_bound=None, + ) + continue + blocker = _come_alive_blocker(level, load.seconds) + options.append( + _Option( + receipt=receipt, + order=order, + warmth=Warmth.LAZY_RESIDENT, + cost=lazy_cost, + come_alive_s=load.seconds, + basis=load.basis, + blockers=warm if blocker is None else (*warm, blocker), + current=self.is_current(identity, receipt, Warmth.LAZY_RESIDENT) + and self.current[identity][1].name == host.name, + host=host.name, + ) + ) feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None + held = None + gaps: tuple[MeasurementRequest, ...] = () + current = self.keepable(identity) + if current is not None and not any(option.current for option in options): + entry, lane = current + if not self.kept_blockers(identity, entry, lane, self.slowest_receipt(identity, lane)): + cost = self.current_cost(identity, entry, lane) + cheaper = [option for option in feasible if cost is not None and round(cost - option.cost, 2) > 0] + chosen = min(cheaper, key=lambda option: option.rank) if cheaper else None + if chosen is None: + held = cost + gaps = self.current_gaps(identity, entry, lane, cost) + gapped = { + (request.cell, request.container_shape, request.warmth) + for request in gaps + if Measurement.BOOT_EVIDENCE in request.measure + } return _Choice( identity=identity, level=level, requests=requests, options=tuple(options), chosen=chosen, - boot_requests=tuple(boot_requests.values()), + boot_requests=tuple( + request + for request in (*boot_requests.values(), *load_requests.values()) + if (request.cell, request.container_shape, request.warmth) not in gapped + ), has_valid_receipt=bool(receipts), + held=held, + gaps=gaps, ) - def receipt_requests(self, choice: _Choice) -> list[MeasurementRequest]: + def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} + measured.update( + request.cell.machine_profile for request in choice.gaps if Measurement.RECEIPT in request.measure + ) bound = choice.level.come_alive_p95_s requests = [] for profile in sorted(self.price_list.machine_profiles, key=lambda price: price.name): @@ -503,7 +712,7 @@ def receipt_requests(self, choice: _Choice) -> list[MeasurementRequest]: cold = accelerator * self.window.cold_container_hours(choice.requests, None) candidates.append((cold, Warmth.COLD_WITH_BOUND)) cost, warmth = min(candidates, key=lambda candidate: (candidate[0], candidate[1] is not Warmth.PINNED)) - saving = None if choice.chosen is None else choice.chosen.cost - cost + saving = None if baseline is None else baseline - cost if saving is not None and saving <= 0: continue requests.append( @@ -527,14 +736,14 @@ def receipt_requests(self, choice: _Choice) -> list[MeasurementRequest]: ) return requests - def boot_requests(self, choice: _Choice) -> list[MeasurementRequest]: - if choice.chosen is None: + @staticmethod + def boot_requests(choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: + if baseline is None: return [] - chosen = choice.chosen return [ - replace(request, saving_bound=chosen.cost - request.cost_bound) + replace(request, saving_bound=baseline - request.cost_bound) for request in choice.boot_requests - if chosen.cost - request.cost_bound > 0 + if baseline - request.cost_bound > 0 ] def replanned_total( @@ -582,13 +791,13 @@ def booted_within(self, identity: str, option: _Option, bound: int) -> tuple[Boo ) def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: - chosen = choice.chosen + baseline = choice.baseline bound = choice.level.come_alive_p95_s - if chosen is None or bound is None: + if baseline is None or bound is None: return [] best: dict[tuple[Cell, ContainerShape, Warmth], _Option] = {} for option in choice.options: - if option.warmth not in COLD_WARMTHS or option.cost >= chosen.cost: + if option.warmth is Warmth.PINNED or option.cost >= baseline: continue if [blocker.field for blocker in option.blockers] != ["comeAliveP95S"]: continue @@ -625,11 +834,10 @@ def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: ) return levers - def next_cheaper(self, choice: _Choice, written: Plan, total: float) -> Option | None: - chosen = choice.chosen - if chosen is None: + def next_cheaper(self, choice: _Choice, baseline: float | None, written: Plan, total: float) -> Option | None: + if baseline is None: return None - cheaper = [option for option in choice.options if not option.feasible and option.cost < chosen.cost] + cheaper = [option for option in choice.options if not option.feasible and option.cost < baseline] if not cheaper: return None cost = max(option.cost for option in cheaper) @@ -693,7 +901,7 @@ def kept_blockers( ) -> tuple[Blocker, ...]: level = self.levels[identity] blockers = [] if receipt is None else list(_warm_blockers(level, receipt)) - if entry.warmth in COLD_WARMTHS and level.come_alive_p95_s is not None: + if entry.warmth in BOOT_WARMTHS and level.come_alive_p95_s is not None: boot = self.boots.get((identity, lane.cell, lane.container_shape, entry.warmth)) if boot is not None and boot.seconds is not None: blocker = _come_alive_blocker(level, boot.seconds) @@ -727,11 +935,22 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: if choice.chosen is None and choice.identity in self.current } staying = {identity: self.current[identity][1].name for identity, option in placed.items() if option.current} - reserved = {lane.name for _, lane in kept.values()} | set(staying.values()) - moving = {identity: option for identity, option in placed.items() if identity not in staying} - names = staying | self.lane_names(moving, reserved) - lanes = {lane.name: self.within_ceiling(lane) for _, lane in kept.values()} - for identity, option in placed.items(): + joining = { + identity: host + for identity, option in placed.items() + if (host := option.host) is not None and identity not in staying + } + reserved = {lane.name for _, lane in kept.values()} | set(staying.values()) | set(joining.values()) + moving = { + identity: option + for identity, option in placed.items() + if identity not in staying and identity not in joining + } + names = staying | joining | self.lane_names(moving, reserved) + carried = [lane for _, lane in kept.values()] + carried.extend(self.current_lanes[name] for name in (*staying.values(), *joining.values())) + lanes = {lane.name: self.within_ceiling(lane) for lane in carried} + for identity, option in moving.items(): lanes[names[identity]] = self.placed_lane(names[identity], identity, option) identities = [entry for entry, _ in kept.values()] identities.extend( @@ -751,12 +970,14 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: requests: list[MeasurementRequest] = [] levers: list[Lever] = [] for choice in choices: - requests.extend(self.receipt_requests(choice)) - requests.extend(self.boot_requests(choice)) + baseline = choice.baseline + requests.extend(choice.gaps) + requests.extend(self.receipt_requests(choice, baseline)) + requests.extend(self.boot_requests(choice, baseline)) nearest = None if price_savings: levers.extend(self.levers(choice, new_plan, total)) - nearest = self.next_cheaper(choice, new_plan, total) + nearest = self.next_cheaper(choice, baseline, new_plan, total) reports.append( self._identity_report( choice, names, kept, costs.get(choice.identity), receipts.get(choice.identity), nearest @@ -778,6 +999,11 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: refusals=tuple(sorted(self.refusals, key=_refusal_rank)), identity_changes=identity_changes, lane_changes=lane_changes, + warm_bound_undeclared=tuple( + identity + for identity in sorted(self.levels) + if not _declares_warm_bound(self.levels[identity]) and self.receipts.get(identity) + ), ) return PlannerResult(plan=new_plan, report=report) @@ -800,7 +1026,7 @@ def _identity_report( ) chosen = choice.chosen if chosen is None: - reason = UnplacedReason.WARM_BOUND_MISSED if choice.has_valid_receipt else UnplacedReason.NO_RECEIPT + reason = choice.unplaced_reason if choice.identity in kept: entry, lane = kept[choice.identity] return replace( @@ -810,6 +1036,7 @@ def _identity_report( monthly_cost=cost, unplaced_reason=reason, blocked_by=self.kept_blockers(choice.identity, entry, lane, receipt), + next_cheaper=next_cheaper, ) return replace(report, unplaced_reason=reason) rate = chosen.receipt.throughput.requests_per_second @@ -910,6 +1137,8 @@ def _request_rank(request: MeasurementRequest) -> tuple[object, ...]: request.cell.gpu_product, _shape_key(request.container_shape), tuple(measurement.value for measurement in request.measure), + request.lane or "", + request.concurrency or 0, ) diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index f285f71bc..42eea8e31 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -30,6 +30,8 @@ class IdentityStatus(StrEnum): class UnplacedReason(StrEnum): NO_RECEIPT = "no-receipt" WARM_BOUND_MISSED = "warm-bound-missed" + WARM_BOUND_UNDECLARED = "warm-bound-undeclared" + CURRENT_UNMEASURED = "current-unmeasured" class ComeAliveBasis(StrEnum): @@ -69,7 +71,7 @@ def _compact(fields: dict[str, object | None]) -> dict[str, object]: @dataclass(frozen=True, slots=True, kw_only=True) class Blocker: field: str - bound: int + bound: int | None measured: float | None def to_dict(self) -> dict[str, object]: @@ -88,6 +90,7 @@ class Option: receipt: SourceRef blocked_by: tuple[Blocker, ...] = () saving: float | None = None + lane: str | None = None def to_dict(self) -> dict[str, object]: return _compact( @@ -95,6 +98,7 @@ def to_dict(self) -> dict[str, object]: "cell": self.cell.to_dict(), "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, + "lane": self.lane, "warmth": self.warmth.value, "monthlyCost": _money(self.monthly_cost), "saving": None if self.saving is None else _money(self.saving), @@ -198,6 +202,8 @@ class MeasurementRequest: cost_bound: float saving_bound: float | None container_shape: ContainerShape | None = None + concurrency: int | None = None + lane: str | None = None def to_dict(self) -> dict[str, object]: return _compact( @@ -205,6 +211,8 @@ def to_dict(self) -> dict[str, object]: "identity": self.identity, "cell": self.cell.to_dict(), "containerShape": None if self.container_shape is None else self.container_shape.to_dict(), + "concurrency": self.concurrency, + "lane": self.lane, "measure": [measurement.value for measurement in self.measure], "warmth": self.warmth.value, "reason": self.reason.value, @@ -308,6 +316,7 @@ class Report: refusals: tuple[Refusal, ...] identity_changes: tuple[IdentityChange, ...] lane_changes: tuple[LaneChange, ...] + warm_bound_undeclared: tuple[str, ...] = () @property def monthly_cost(self) -> float: @@ -349,6 +358,7 @@ def to_dict(self) -> dict[str, object]: "measurementRequests": [request.to_dict() for request in self.measurement_requests], "comeAliveLevers": [lever.to_dict() for lever in self.come_alive_levers], "queueWaitUnattained": list(self.queue_wait_unattained), + "warmBoundUndeclared": list(self.warm_bound_undeclared), "refusals": [refusal.to_dict() for refusal in self.refusals], "changes": { "identities": [change.to_dict() for change in self.identity_changes], diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 5d4325782..076b55a12 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -392,57 +392,25 @@ "reason": "unmeasured", "costBound": 0.0 }, - { - "identity": "intfloat/e5-base-v2", - "cell": { - "machineProfile": "a100-80gb", - "gpuProduct": "NVIDIA-A100-SXM4-80GB" - }, - "measure": [ - "receipt" - ], - "warmth": "pinned", - "reason": "unmeasured", - "costBound": 1825.0 - }, - { - "identity": "intfloat/e5-base-v2", - "cell": { - "machineProfile": "h100", - "gpuProduct": "NVIDIA-H100-80GB-HBM3" - }, - "measure": [ - "receipt" - ], - "warmth": "pinned", - "reason": "unmeasured", - "costBound": 2920.0 - }, { "identity": "intfloat/e5-base-v2", "cell": { "machineProfile": "l4", "gpuProduct": "NVIDIA-L4" }, - "measure": [ - "receipt" - ], - "warmth": "pinned", - "reason": "unmeasured", - "costBound": 584.0 - }, - { - "identity": "intfloat/e5-base-v2", - "cell": { - "machineProfile": "l40s", - "gpuProduct": "NVIDIA-L40S" + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 }, + "concurrency": 16, + "lane": "default-l4-default", "measure": [ "receipt" ], "warmth": "pinned", "reason": "unmeasured", - "costBound": 876.0 + "costBound": 759.2 }, { "identity": "Qwen/Qwen3-Embedding-4B", @@ -492,6 +460,27 @@ "reason": "unmeasured", "costBound": 146.0, "savingBound": 236.03 + }, + { + "identity": "BAAI/bge-m3:sparse", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "lane": "default-l4-default", + "measure": [ + "boot-evidence" + ], + "warmth": "lazy-resident", + "reason": "unmeasured", + "costBound": 0.0, + "savingBound": 47.45 } ], "comeAliveLevers": [ @@ -541,6 +530,7 @@ "BAAI/bge-m3:sparse", "intfloat/e5-base-v2" ], + "warmBoundUndeclared": [], "refusals": [ { "record": "boot-evidence", diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index de33e77a4..463ad80a8 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -66,6 +66,8 @@ SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) WIDE = ContainerShape(gpu_count=1, cpu_cores=8, memory_mib=32768) IDENTITY = "org/model" +ANY_P99_MS = 1000 +WARM = ServiceLevel(warm_p99_ms=ANY_P99_MS) ESTATE_PARAMETERS = PlannerParameters( placement="us-central1", @@ -154,6 +156,8 @@ def levels(*entries: tuple[str, ServiceLevel]) -> ServiceLevels: def level(**fields) -> ServiceLevels: + if not fields.keys() & {"warm_p99_ms", "ttft_p99_ms", "itl_p99_ms"}: + fields["warm_p99_ms"] = ANY_P99_MS return levels((IDENTITY, ServiceLevel(**fields))) @@ -224,6 +228,51 @@ def single_lane_plan( ) +ANCHOR = "org/anchor" + + +def plan_lane(name, *, cell=L4, floor=1, bundle="default", queue_pool="default") -> PlanLane: + return PlanLane( + name=name, + queue_pool=queue_pool, + bundle=bundle, + cell=cell, + container_shape=SHAPE, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=floor, floor_max=max(floor, 1)), + ) + + +def plan_of(lanes, residents) -> Plan: + return Plan( + placement="test", + lanes=tuple(lanes), + identities=tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) + for identity, (name, warmth) in sorted(residents.items()) + ), + ) + + +def resident_levels(bound=60, anchor_bound=0) -> ServiceLevels: + return levels( + (ANCHOR, ServiceLevel(come_alive_p95_s=anchor_bound, warm_p99_ms=ANY_P99_MS)), + (IDENTITY, ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=ANY_P99_MS)), + ) + + +def resident_estate() -> Plan: + return plan_of( + [plan_lane("own"), plan_lane("shared")], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + + +RESIDENT_RECEIPTS = [receipt(identity=ANCHOR), receipt()] +LOADS = [boot(warmth=Warmth.LAZY_RESIDENT, seconds=20.0, uri="loads/example.json")] + + def load(name: str): return json.loads((FIXTURES / name).read_text(encoding="utf-8")) @@ -431,7 +480,7 @@ def test_a_failed_boot_disqualifies_the_successful_ones_beside_it(): assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED -def test_the_slowest_successful_boot_stands_for_the_percentile(): +def test_fewer_than_twenty_boots_stand_for_the_come_alive_at_their_slowest(): boots = [boot(seconds=80.0, uri="boots/1.json"), boot(seconds=130.0, uri="boots/2.json")] result = run(level(come_alive_p95_s=120), [receipt()], boots, prices=L4_ONLY) report = result.report.identity(IDENTITY) @@ -439,6 +488,30 @@ def test_the_slowest_successful_boot_stands_for_the_percentile(): assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=120, measured=130.0),) +@pytest.mark.parametrize( + ("seconds", "come_alive"), + [ + ([50.0] * 18 + [130.0], 130.0), + ([50.0] * 19 + [130.0], 50.0), + ([50.0] * 37 + [130.0] * 3, 130.0), + ([50.0] * 38 + [130.0] * 2, 50.0), + ], +) +def test_twenty_or_more_boots_stand_for_the_come_alive_at_their_95th_percentile(seconds, come_alive): + boots = [boot(seconds=value, uri=f"boots/{index}.json") for index, value in enumerate(seconds)] + result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, come_alive) + assert rendered(run(level(come_alive_p95_s=600), [receipt()], boots[::-1], prices=L4_ONLY)) == rendered(result) + + +def test_a_failed_boot_withdraws_the_warmth_however_many_boots_succeeded(): + boots = [boot(seconds=50.0, uri=f"boots/{index}.json") for index in range(30)] + boots.append(boot(seconds=0.0, outcome=BootOutcome.FAILED, uri="boots/failed.json")) + result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + + def test_boot_seconds_render_the_same_whatever_their_number_type(): boots = [boot(seconds=60, uri="boots/1.json"), boot(seconds=60.0, uri="boots/2.json")] first = run(level(come_alive_p95_s=120), [receipt()], boots, prices=L4_ONLY) @@ -617,8 +690,8 @@ def test_a_cell_that_cannot_save_is_not_requested(): def test_measurement_requests_rank_unplaced_identities_first_then_by_saving(): service_levels = levels( - ("org/big", ServiceLevel(come_alive_p95_s=0)), - ("org/small", ServiceLevel(come_alive_p95_s=120)), + ("org/big", ServiceLevel(come_alive_p95_s=0, warm_p99_ms=ANY_P99_MS)), + ("org/small", ServiceLevel(come_alive_p95_s=120, warm_p99_ms=ANY_P99_MS)), ("org/unplaced", ServiceLevel(come_alive_p95_s=0)), ) receipts = [receipt(identity="org/big", cell=A100), receipt(identity="org/small", cell=L4)] @@ -847,12 +920,12 @@ def test_an_identity_without_a_feasible_cell_or_a_current_lane_is_left_out(): def test_lane_names_derive_from_the_identity_and_its_machine_profile(): - result = run(levels(("BAAI/bge-m3:sparse", ServiceLevel())), [receipt(identity="BAAI/bge-m3:sparse")]) + result = run(levels(("BAAI/bge-m3:sparse", WARM)), [receipt(identity="BAAI/bge-m3:sparse")]) assert result.plan.lanes[0].name == "baai-bge-m3-sparse-l4" def test_colliding_lane_names_take_an_identity_digest(): - service_levels = levels(("org/model", ServiceLevel()), ("org-model", ServiceLevel())) + service_levels = levels(("org/model", WARM), ("org-model", WARM)) receipts = [receipt(identity="org/model"), receipt(identity="org-model")] result = run(service_levels, receipts, prices=L4_ONLY) names = {entry.identity: entry.lane for entry in result.plan.identities} @@ -864,7 +937,7 @@ def test_colliding_lane_names_take_an_identity_digest(): def test_a_derived_lane_name_never_takes_a_kept_lane_name(): current = single_lane_plan(name="org-model-l4", identity="org/kept") - service_levels = levels(("org/kept", ServiceLevel(come_alive_p95_s=0)), (IDENTITY, ServiceLevel())) + service_levels = levels(("org/kept", ServiceLevel(come_alive_p95_s=0)), (IDENTITY, WARM)) result = run(service_levels, [receipt()], prices=L4_ONLY, current=current) names = {entry.identity: entry.lane for entry in result.plan.identities} assert names["org/kept"] == "org-model-l4" @@ -879,31 +952,263 @@ def test_a_current_placement_keeps_its_lane_name(): assert result.report.unchanged -def test_a_current_placement_that_differs_only_in_its_envelope_keeps_its_lane_name(): +def test_a_current_placement_keeps_its_lane_as_it_is_under_the_reviewed_ceiling(): current = single_lane_plan(name="legacy", max_containers=8, envelope=Envelope(floor_min=2, floor_max=5)) - result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current) + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current, buffer_containers=2) (lane,) = result.plan.lanes - assert (lane.name, lane.max_containers, lane.envelope) == ("legacy", 4, Envelope(floor_min=1, floor_max=1)) + assert (lane.name, lane.max_containers, lane.envelope) == ("legacy", 4, Envelope(floor_min=2, floor_max=4)) + assert (lane.cell, lane.container_shape, lane.concurrency) == (L4, SHAPE, 8) + assert result.report.identity(IDENTITY).status is IdentityStatus.PLACED assert result.report.identity_changes == () assert [change.name for change in result.report.lane_changes] == ["legacy"] + within = single_lane_plan(name="legacy", max_containers=4, envelope=Envelope(floor_min=1, floor_max=3)) + assert run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=within).plan == within -def test_a_current_lane_that_differs_in_concurrency_is_not_the_current_placement(): +def test_a_receipt_at_the_current_concurrency_keeps_the_current_lane(): current = single_lane_plan(name="legacy", max_containers=4) receipts = [receipt(concurrency=16), receipt(concurrency=8)] result = run(level(come_alive_p95_s=0), receipts, prices=L4_ONLY, current=current) (lane,) = result.plan.lanes assert (lane.name, lane.concurrency) == ("legacy", 8) assert result.report.unchanged - moved = run(level(come_alive_p95_s=0), [receipts[0]], prices=L4_ONLY, current=current) - (lane,) = moved.plan.lanes - assert (lane.name, lane.concurrency) == ("org-model-l4", 16) - assert [(change.name, change.before is None) for change in moved.report.lane_changes] == [ - ("legacy", False), - ("org-model-l4", True), + + +def test_a_current_lane_without_a_receipt_at_its_concurrency_is_kept_and_asks_for_one(): + current = single_lane_plan(name="legacy", max_containers=4) + result = run(level(come_alive_p95_s=0), [receipt(concurrency=16)], prices=L4_ONLY, current=current) + assert result.plan == current + assert result.report.unchanged + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + (request,) = result.report.measurement_requests + assert (request.cell, request.container_shape, request.concurrency, request.lane) == (L4, SHAPE, 8, "legacy") + assert (request.measure, request.warmth, request.reason) == ( + (Measurement.RECEIPT,), + Warmth.PINNED, + RequestReason.UNMEASURED, + ) + assert (request.cost_bound, request.saving_bound) == (pytest.approx(MONTH), None) + assert request.to_dict()["lane"] == "legacy" + assert request.to_dict()["concurrency"] == 8 + + +def test_a_cold_lane_without_boot_evidence_is_kept_and_asks_for_it(): + current = single_lane_plan(name="cold", warmth=Warmth.COLD_WITH_BOUND) + result = run(level(come_alive_p95_s=120), [receipt()], prices=L4_ONLY, hourly={IDENTITY: {0: 1}}, current=current) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + (request,) = result.report.measurement_requests + assert (request.measure, request.warmth, request.lane, request.concurrency, request.reason) == ( + (Measurement.BOOT_EVIDENCE,), + Warmth.COLD_WITH_BOUND, + "cold", + 8, + RequestReason.UNMEASURED, + ) + assert request.cost_bound == pytest.approx((1 / 10.0 + 1 * 2 * 900) / 3600 * SCALE) + + +def test_an_unmeasured_current_lane_moves_to_a_measured_option_that_is_cheaper(): + current = single_lane_plan(name="cold", warmth=Warmth.COLD_WITH_BOUND, cell=A100) + result = run( + level(come_alive_p95_s=120), + [receipt(cell=A100), receipt()], + [boot()], + hourly={IDENTITY: {0: 1}}, + current=current, + ) + report = result.report.identity(IDENTITY) + assert (report.status, report.chosen.cell, report.chosen.warmth) == ( + IdentityStatus.PLACED, + L4, + Warmth.COLD_WITH_BOUND, + ) + assert [entry.lane for entry in result.plan.identities] == ["org-model-l4"] + + +def test_a_cold_current_lane_under_a_zero_bound_moves_to_a_warm_floor_without_evidence(): + current = single_lane_plan(name="cold", warmth=Warmth.COLD_WITH_BOUND) + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current) + assert [(entry.lane, entry.warmth) for entry in result.plan.identities] == [("org-model-l4", Warmth.PINNED)] + assert result.report.measurement_requests == () + + +def test_an_undeclared_warm_bound_keeps_the_current_class_and_is_reported(): + current = single_lane_plan(cell=A100) + receipts = [receipt(cell=A100), receipt(p99=300.0)] + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), receipts, current=current) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert (report.status, report.chosen.cell) == (IdentityStatus.PLACED, A100) + assert report.next_cheaper.cell == L4 + assert report.next_cheaper.blocked_by == (Blocker(field="warmP99Ms", bound=None, measured=300.0),) + assert report.next_cheaper.to_dict()["blockedBy"] == [{"field": "warmP99Ms", "measured": 300.0}] + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + assert result.report.warm_bound_undeclared == (IDENTITY,) + assert result.report.to_dict()["warmBoundUndeclared"] == [IDENTITY] + + +def test_an_undeclared_generation_bound_blocks_both_token_latencies_off_the_current_class(): + current = single_lane_plan(cell=A100, concurrency=16) + receipts = [generation_receipt(cell=A100), generation_receipt(cell=L4, ttft=2000.0, itl=95.0)] + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), receipts, current=current) + report = result.report.identity(IDENTITY) + assert report.chosen.cell == A100 + assert report.next_cheaper.blocked_by == ( + Blocker(field="ttftP99Ms", bound=None, measured=2000.0), + Blocker(field="itlP99Ms", bound=None, measured=95.0), + ) + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + + +def test_an_undeclared_warm_bound_without_a_current_class_places_nothing(): + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), [receipt()], prices=L4_ONLY) + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.UNPLACED, UnplacedReason.WARM_BOUND_UNDECLARED) + assert result.plan.identities == () + assert result.report.warm_bound_undeclared == (IDENTITY,) + + +def test_an_undeclared_warm_bound_without_a_receipt_on_the_current_class_keeps_it(): + current = single_lane_plan(cell=A100) + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), [receipt()], current=current) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.WARM_BOUND_UNDECLARED) + assert [(request.cell, request.measure, request.lane) for request in result.report.measurement_requests] == [ + (A100, (Measurement.RECEIPT,), "shared") + ] + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + + +def test_an_identity_without_any_bound_has_no_undeclared_warm_bound_until_it_has_a_cell(): + result = run(levels((IDENTITY, ServiceLevel())), [], prices=L4_ONLY) + assert result.report.warm_bound_undeclared == () + + +def test_a_lazy_resident_without_evidence_keeps_its_place_and_asks_for_it(): + current = plan_of( + [plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("shared", Warmth.LAZY_RESIDENT)} + ) + receipts = [receipt(identity=ANCHOR), receipt(concurrency=16)] + result = run( + resident_levels(), receipts, [boot(seconds=30.0)], prices=L4_ONLY, hourly={IDENTITY: {0: 10}}, current=current + ) + assert result.plan == current + assert result.report.unchanged + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason, report.lane) == ( + IdentityStatus.KEPT, + UnplacedReason.CURRENT_UNMEASURED, + "shared", + ) + requests = [ + (request.measure, request.warmth, request.lane, request.concurrency, request.saving_bound) + for request in result.report.measurement_requests + if request.identity == IDENTITY + ] + assert requests == [ + ((Measurement.BOOT_EVIDENCE,), Warmth.LAZY_RESIDENT, "shared", 8, None), + ((Measurement.RECEIPT,), Warmth.LAZY_RESIDENT, "shared", 8, None), ] +def test_a_measured_load_onto_a_lane_a_never_cold_neighbour_pins_makes_a_lazy_resident(): + current = resident_estate() + hourly = {IDENTITY: {3: 90000}} + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, hourly=hourly, current=current) + report = result.report.identity(IDENTITY) + assert (report.status, report.lane, report.chosen.warmth, report.chosen.lane) == ( + IdentityStatus.PLACED, + "shared", + Warmth.LAZY_RESIDENT, + "shared", + ) + assert (report.chosen.come_alive_s, report.chosen.come_alive_basis) == (20.0, ComeAliveBasis.BOOT_EVIDENCE) + assert report.chosen.monthly_cost == pytest.approx(2 * SCALE) + assert report.chosen.to_dict()["lane"] == "shared" + assert result.plan.lanes == (current.lanes[1],) + assert [(entry.identity, entry.lane, entry.warmth) for entry in result.plan.identities] == [ + (ANCHOR, "shared", Warmth.PINNED), + (IDENTITY, "shared", Warmth.LAZY_RESIDENT), + ] + assert result.report.monthly_cost == pytest.approx(MONTH + 2 * SCALE) + again = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, hourly=hourly, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_an_identity_pinned_beside_a_never_cold_neighbour_stays_pinned_there(): + current = plan_of([plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("shared", Warmth.PINNED)}) + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + assert result.plan == current + assert result.report.unchanged + assert result.report.come_alive_levers == () + + +def test_a_load_over_the_bound_is_a_come_alive_lever(): + result = run(resident_levels(bound=10), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=resident_estate()) + report = result.report.identity(IDENTITY) + assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=10, measured=20.0),) + (lever,) = result.report.come_alive_levers + assert (lever.identity, lever.warmth, lever.permitted_bound_s) == (IDENTITY, Warmth.LAZY_RESIDENT, 20) + assert lever.saving_at_permitted_bound == pytest.approx(MONTH) + assert lever.saving_under_bound == pytest.approx(MONTH) + + +def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_from_the_next_run(): + current = plan_of([plan_lane("own")], {IDENTITY: ("own", Warmth.PINNED)}) + first = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + assert [(entry.identity, entry.lane, entry.warmth) for entry in first.plan.identities] == [ + (ANCHOR, "org-anchor-l4", Warmth.PINNED), + (IDENTITY, "own", Warmth.PINNED), + ] + second = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=first.plan) + assert [(entry.identity, entry.lane, entry.warmth) for entry in second.plan.identities] == [ + (ANCHOR, "org-anchor-l4", Warmth.PINNED), + (IDENTITY, "org-anchor-l4", Warmth.LAZY_RESIDENT), + ] + assert first.report.monthly_cost - second.report.monthly_cost == pytest.approx(MONTH) + third = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=second.plan) + assert third.report.unchanged + + +@pytest.mark.parametrize( + ("shared", "anchor_bound"), + [ + (plan_lane("shared"), 30), + (plan_lane("shared", bundle="sglang"), 0), + (plan_lane("shared", queue_pool="batch"), 0), + ], +) +def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_lane_of_its_pool_and_bundle(shared, anchor_bound): + current = plan_of([plan_lane("own"), shared], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}) + result = run(resident_levels(anchor_bound=anchor_bound), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + report = result.report.identity(IDENTITY) + assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert all(lever.warmth is not Warmth.LAZY_RESIDENT for lever in result.report.come_alive_levers) + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + + +@pytest.mark.parametrize( + ("bound", "expected"), [(60, [(Warmth.COLD_WITH_BOUND, None), (Warmth.LAZY_RESIDENT, "shared")]), (0, [])] +) +def test_a_warm_lane_without_a_load_row_asks_for_one_at_the_saving_it_unlocks(bound, expected): + result = run(resident_levels(bound=bound), RESIDENT_RECEIPTS, prices=L4_ONLY, current=resident_estate()) + requests = [request for request in result.report.measurement_requests if request.identity == IDENTITY] + assert [(request.warmth, request.lane) for request in requests] == expected + for request in requests: + assert (request.measure, request.reason, request.cost_bound) == ( + (Measurement.BOOT_EVIDENCE,), + RequestReason.UNMEASURED, + 0, + ) + assert request.saving_bound == pytest.approx(MONTH) + + def test_a_current_placement_on_a_shared_lane_stays_beside_its_kept_neighbour(): current = single_lane_plan(others=("org/kept",), max_containers=4) service_levels = levels( @@ -958,10 +1263,10 @@ def test_a_new_queue_pool_or_bundle_moves_the_identity_off_its_current_lane(over def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): - service_levels = levels(("org/model", ServiceLevel()), ("org-model", ServiceLevel())) + service_levels = levels(("org/model", WARM), ("org-model", WARM)) receipts = [receipt(identity="org/model"), receipt(identity="org-model")] first = run(service_levels, receipts, prices=L4_ONLY) - second = run(levels(("org/model", ServiceLevel())), receipts, prices=L4_ONLY, current=first.plan) + second = run(levels(("org/model", WARM)), receipts, prices=L4_ONLY, current=first.plan) digest = hashlib.sha256(b"org-model").hexdigest()[:8] kept_name = f"org-model-l4-{hashlib.sha256(b'org/model').hexdigest()[:8]}" assert [entry.lane for entry in second.plan.identities] == [kept_name] @@ -970,7 +1275,7 @@ def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): def test_plan_lanes_that_name_one_chart_lane_are_reported(): - service_levels = levels(("org/a", ServiceLevel()), ("org/b", ServiceLevel()), ("org/c", ServiceLevel())) + service_levels = levels(("org/a", WARM), ("org/b", WARM), ("org/c", WARM)) receipts = [receipt(identity=identity) for identity in ("org/a", "org/b", "org/c")] result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/c": "sglang"}) (shared,) = result.report.shared_chart_lanes @@ -992,7 +1297,7 @@ def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): placement="test", lanes=(*first.lanes, *second.lanes), identities=(*first.identities, *second.identities) ) service_levels = levels( - (IDENTITY, ServiceLevel()), + (IDENTITY, WARM), ("org/kept", ServiceLevel(come_alive_p95_s=0)), ("org/kept-too", ServiceLevel(come_alive_p95_s=0)), ) @@ -1007,7 +1312,7 @@ def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): def test_bundles_and_the_queue_pool_come_from_the_parameters(): - service_levels = levels(("org/embed", ServiceLevel()), ("org/generate", ServiceLevel())) + service_levels = levels(("org/embed", WARM), ("org/generate", WARM)) result = run( service_levels, [receipt(identity="org/embed"), receipt(identity="org/generate")], @@ -1164,6 +1469,22 @@ def test_the_estate_report_states_its_findings(): assert (kept.status, kept.lane, kept.monthly_cost) == (IdentityStatus.KEPT, "default-l4-default", None) assert [refusal.record for refusal in report.refusals] == [RefusedRecord.BOOT_EVIDENCE, RefusedRecord.RECEIPT] assert report.queue_wait_unattained == ("BAAI/bge-m3", "BAAI/bge-m3:sparse", "intfloat/e5-base-v2") + assert report.warm_bound_undeclared == () + on_lanes = [ + (request.identity, request.measure, request.warmth, request.lane, request.saving_bound) + for request in report.measurement_requests + if request.lane is not None + ] + assert on_lanes == [ + ("intfloat/e5-base-v2", (Measurement.RECEIPT,), Warmth.PINNED, "default-l4-default", None), + ( + "BAAI/bge-m3:sparse", + (Measurement.BOOT_EVIDENCE,), + Warmth.LAZY_RESIDENT, + "default-l4-default", + pytest.approx(sparse.chosen.monthly_cost), + ), + ] assert [change.identity for change in report.identity_changes] == ["BAAI/bge-m3:sparse", "Qwen/Qwen3-Embedding-4B"] assert report.monthly_cost == pytest.approx(sum(lane.monthly_cost for lane in report.lanes)) shared = {lane.name: lane for lane in report.lanes}["default-l4-default"] @@ -1328,3 +1649,148 @@ def test_random_estates_choose_only_feasible_cells_and_replan_to_themselves(seed rng.shuffle(shuffled) reordered = run(service_levels, shuffled, list(reversed(boots)), prices=prices, hourly=hourly, **overrides) assert rendered(reordered) == rendered(result) + + +def capped(lane: PlanLane, ceiling: int) -> PlanLane: + if lane.max_containers <= ceiling: + return lane + envelope = replace( + lane.envelope, + floor_min=min(lane.envelope.floor_min, ceiling), + floor_max=min(lane.envelope.floor_max, ceiling), + ) + return replace(lane, max_containers=ceiling, envelope=envelope) + + +@pytest.mark.parametrize("seed", range(40)) +def test_random_current_estates_keep_what_no_measured_option_beats(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = [f"org/model-{index}" for index in range(rng.randint(2, 6))] + lanes = [] + for index in range(rng.randint(1, 3)): + floor = 1 if index == 0 or rng.random() < 0.6 else 0 + ceiling = rng.choice([2, 4, 6]) + lanes.append( + PlanLane( + name=f"lane-{index}", + queue_pool="default", + bundle="default", + cell=rng.choice([L4, A100]), + container_shape=rng.choice([SHAPE, WIDE]), + concurrency=rng.choice([4, 8]), + max_containers=ceiling, + envelope=Envelope(floor_min=floor, floor_max=min(ceiling, floor + rng.randint(0, 2))), + ) + ) + anchor_lane = lanes[0] + residents = {identities[0]: (anchor_lane.name, Warmth.PINNED)} + for identity in identities[1:]: + if rng.random() < 0.8: + host = rng.choice(lanes) + if host.envelope.floor_min: + warmth = rng.choice([Warmth.PINNED, Warmth.PINNED, Warmth.LAZY_RESIDENT]) + else: + warmth = rng.choice([Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD]) + residents[identity] = (host.name, warmth) + hosted = {name for name, _ in residents.values()} + current = plan_of([lane for lane in lanes if lane.name in hosted], residents) + service_levels = levels( + *( + ( + identity, + ServiceLevel( + come_alive_p95_s=0 if identity == identities[0] else rng.choice([None, 0, 30, 120, 600]), + warm_p99_ms=rng.choice([None, 150, 250, 400]), + ), + ) + for identity in identities + ) + ) + on_anchor = (anchor_lane.cell, anchor_lane.container_shape, anchor_lane.concurrency) + receipts = [ + receipt( + identity=identity, + cell=cell, + shape=shape, + concurrency=concurrency, + rps=rng.choice([0.5, 5.0, 50.0]), + p99=rng.choice([100.0, 200.0, 300.0, 500.0]), + uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}", + ) + for identity in identities + for cell in (L4, A100) + for shape in (SHAPE, WIDE) + for concurrency in (4, 8) + if rng.random() < (0.6 if (cell, shape, concurrency) == on_anchor else 0.3) + ] + boots = [ + boot( + identity=identity, + cell=cell, + warmth=warmth, + seconds=rng.choice([5.0, 60.0, 200.0]), + outcome=rng.choice([BootOutcome.SUCCEEDED, BootOutcome.SUCCEEDED, BootOutcome.FAILED]), + shape=shape, + uri=f"boots/{identity}/{cell.machine_profile}/{shape.memory_mib}/{warmth.value}", + ) + for identity in identities + for cell in (L4, A100) + for shape in (SHAPE, WIDE) + for warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD, Warmth.LAZY_RESIDENT) + if rng.random() < (0.6 if (cell, shape, warmth) == (*on_anchor[:2], Warmth.LAZY_RESIDENT) else 0.3) + ] + hourly = { + identity: {hour: rng.choice([1, 50, 20000]) for hour in rng.sample(range(WINDOW_HOURS), rng.randint(0, 4))} + for identity in identities + } + prices = price_list(l4=rng.choice([0.5, 1.0]), a100=rng.choice([2.0, 3.0]), core=0.1) + overrides = {"buffer_containers": rng.randint(0, 2), "fallback_floor": rng.randint(0, 3)} + result = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=current, **overrides) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + before = {lane.name: lane for lane in current.lanes} + after = {lane.name: lane for lane in result.plan.lanes} + for name, lane in after.items(): + if name in before: + assert lane == capped(before[name], 4) + placements = {entry.identity: entry for entry in result.plan.identities} + for entry in result.report.identities: + level = entry.service_level + was = residents.get(entry.identity) + if entry.status is IdentityStatus.KEPT: + assert (placements[entry.identity].lane, placements[entry.identity].warmth) == was + continue + declared = level.warm_p99_ms is not None + if entry.chosen is None: + assert entry.identity not in placements + continue + chosen = entry.chosen + assert chosen.blocked_by == () + if not declared: + assert was is not None + assert chosen.cell == before[was[0]].cell + if chosen.warmth is not Warmth.PINNED: + assert level.come_alive_p95_s != 0 + assert level.come_alive_p95_s is None or chosen.come_alive_s <= level.come_alive_p95_s + if chosen.warmth is Warmth.LAZY_RESIDENT: + assert chosen.lane in before + assert before[chosen.lane].envelope.floor_min >= 1 + if entry.next_cheaper is not None: + looser = relaxed(service_levels, entry.identity, entry.next_cheaper.blocked_by) + again = run(looser, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) + fall = result.report.monthly_cost - again.report.monthly_cost + assert entry.next_cheaper.saving == pytest.approx(max(0.0, fall), abs=0.01) + shuffled = list(receipts) + rng.shuffle(shuffled) + reordered = run( + ServiceLevels(identities=tuple(reversed(service_levels.identities))), + shuffled, + list(reversed(boots)), + prices=prices, + hourly=hourly, + current=current, + **overrides, + ) + assert rendered(reordered) == rendered(result) + again = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) + assert again.plan == result.plan + assert again.report.unchanged From c469b250b423d6e28b170a5720ab5272d2cb950a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 13:27:01 +0100 Subject: [PATCH 10/40] docs(placement): rewrap the lazy-resident selection step --- packages/sie_placement/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index bc24b2f6c..d0cf747c3 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -163,10 +163,10 @@ document: with a `floorMin` of at least 1, on the parameters' queue pool and the identity's bundle, on which another identity whose `comeAliveP95S` is `0` is pinned. The identity's own current lane is a host only when it is a lazy - resident there, since its floor is paid either way. - The come-alive is the identity's `lazy-resident` boot evidence, its measured - load onto that cell and container shape, under the same statistic. The - option costs what the identity's demand needs above the host lane's floor. + resident there, since its floor is paid either way. The come-alive is the + identity's `lazy-resident` boot evidence, its measured load onto that cell + and container shape, under the same statistic. The option costs what the + identity's demand needs above the host lane's floor. 5. The cheapest unblocked option wins. Ties go to the current placement, then to the faster come-alive. From 195cd2ef0d593106ec97b893b1b4f4efab1ec406 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 14:08:28 +0100 Subject: [PATCH 11/40] fix(placement): decide lazy hosts against the plan being written A lazy resident's option was hosted by any lane of the current plan on which a never-cold identity was pinned, even when that identity moved in the same run. The resident then sat alone on a warm-floor lane while its option claimed to cost nothing beyond a floor nobody else paid. Identities whose comeAliveP95S is 0 are now placed first, since no cold or lazy option can meet that bound, and a lane hosts lazy residents only when one of them stays pinned on it in this run. A resident's own lazy lane without such a neighbour stays an option, priced at the lane's floor as well. A boot or load at the current lane that did not succeed now blocks the current placement instead of holding it as unmeasured, and a current placement kept on a lane the price list cannot price reports current-unpriced rather than current-unmeasured. --- packages/sie_placement/README.md | 59 ++++---- .../src/sie_placement/planner.py | 61 ++++---- .../sie_placement/src/sie_placement/report.py | 1 + packages/sie_placement/tests/test_planner.py | 133 +++++++++++++++++- 4 files changed, 201 insertions(+), 53 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index d0cf747c3..7a8be6e86 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -123,9 +123,11 @@ print(render(result.plan.to_dict()), render(result.report.to_dict())) same bytes in whatever order the receipts, boot evidence, identities and demand series arrive. An identity that stays on its current placement keeps its current lane as it is, so a run whose `current_plan` already holds every -choice, such as its own previous plan, reports no change. The one exception is -a lane that the previous run created for an identity whose `comeAliveP95S` is -`0`: it hosts lazy residents only from the run after it. +choice, such as its own previous plan, reports no change. The exceptions are +lazy residents that join a host lane one run later: a lane the previous run +created for an identity whose `comeAliveP95S` is `0` hosts lazy residents only +from the run after it, and an identity that left a host lane it was pinned on +may rejoin that lane as a lazy resident on the next run. | Parameter | Meaning | | --- | --- | @@ -161,12 +163,16 @@ document: 4. A cell also offers `lazy-resident` on each host lane with its cell, container shape and concurrency. A host lane is a lane of the current plan with a `floorMin` of at least 1, on the parameters' queue pool and the - identity's bundle, on which another identity whose `comeAliveP95S` is `0` is - pinned. The identity's own current lane is a host only when it is a lazy - resident there, since its floor is paid either way. The come-alive is the + identity's bundle, on which another identity whose `comeAliveP95S` is `0` + stays pinned in this run, on its current placement or kept there. Those + identities are placed first: a bound of `0` admits no cold or lazy option, + so no other choice moves them. The identity's own current lane is also a + host when it is a lazy resident there already. The come-alive is the identity's `lazy-resident` boot evidence, its measured load onto that cell and container shape, under the same statistic. The option costs what the - identity's demand needs above the host lane's floor. + identity's demand needs above the host lane's floor, plus the floor itself + on a lane where no such identity stays pinned, since no other identity pays + it. 5. The cheapest unblocked option wins. Ties go to the current placement, then to the faster come-alive. @@ -175,16 +181,19 @@ option's cell, container shape and concurrency, the identity's warmth is the option's, and the lane's `queuePool` and `bundle` are the ones the parameters give the identity; a `lazy-resident` option must also be hosted by that lane. A current placement that no option matches lacks evidence: a receipt at its -lane's cell, container shape and concurrency, a successful boot for a cold -warmth, or a load for a lazy resident. Unless it is known to miss a bound, -through a receipt at the lane that misses a warm bound, a measured come-alive -over the bound, or a cold or `lazy-resident` warmth under a bound of `0`, the -identity keeps that placement and leaves it only for an unblocked option at -least a cent cheaper than the placement. The placement is priced as an option -of its warmth at the lane's fastest receipt, or without one at its floor alone: -one container all month for `pinned`, nothing beyond the host lane's floor for -`lazy-resident`, and the burst tails for a cold warmth. What the placement -lacks becomes a measurement request. +lane's cell, container shape and concurrency, a boot for a cold warmth, or a +load for a lazy resident. Unless it is known to miss a bound, through a receipt +at the lane that misses a warm bound, a measured come-alive over the bound, a +boot or load at the lane that did not succeed, or a cold or `lazy-resident` +warmth under a bound of `0`, the identity keeps that placement and leaves it +only for an unblocked option at least a cent cheaper than the placement. The +placement is priced as an option of its warmth at the lane's fastest receipt, +or without one at its floor alone: one container all month for `pinned`, for +`lazy-resident` nothing beyond the lane's floor, or the floor itself where no +identity whose `comeAliveP95S` is `0` stays pinned on the lane, and the burst +tails for a cold warmth. What the placement lacks becomes a measurement +request. Such a placement on a lane the price list cannot price has no cost to +compare, so the identity keeps it. An identity that stays on its current placement keeps that lane exactly as it is, beside whoever else the lane hosts: its cell, container shape, @@ -251,12 +260,14 @@ The report renders as a `placement-report` document: total would not fall, as for an identity whose lane keeps its floor for its other residents. An identity without a placement names its `unplacedReason`: `no-receipt`, `warm-bound-missed`, - `warm-bound-undeclared` when no cell is on its current class, or + `warm-bound-undeclared` when no cell is on its current class, `current-unmeasured` when it keeps a current placement that lacks evidence - and no unblocked option is cheaper. A kept identity's `blockedBy` names the - bounds its kept placement is known to miss: its receipt's warm latencies, a - measured come-alive over the bound, or a cold or `lazy-resident` warmth under - a bound of `0`, which needs no measurement. + and no unblocked option is cheaper, or `current-unpriced` when it keeps a + current placement on a lane the price list cannot price. A kept identity's + `blockedBy` names the bounds its kept placement is known to miss: its + receipt's warm latencies, a measured come-alive over the bound, and with no + `measured` value a boot or load at its lane that did not succeed, or a cold + or `lazy-resident` warmth under a bound of `0`, which needs no measurement. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, @@ -269,8 +280,8 @@ The report renders as a `placement-report` document: accelerator at its rate, kept warm or, where the come-alive bound allows, paying burst tails only. A cell whose receipt has no successful boot asks for boot evidence and is bounded by its cold cost, and a host lane without the - identity's load asks for `lazy-resident` boot evidence on it, bounded by the - demand above the host's floor. Savings are measured against the choice, or + identity's load asks for `lazy-resident` boot evidence on it, bounded by + what that option would cost. Savings are measured against the choice, or against a kept identity's current placement. A request that cannot save is dropped, and the requests of an identity without a placement or a current one carry no bound and rank first. The evidence a kept current placement diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 88ed83bcb..b3e82d23b 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -249,6 +249,7 @@ class _Choice: has_valid_receipt: bool held: float | None = None gaps: tuple[MeasurementRequest, ...] = () + unpriced: bool = False @property def baseline(self) -> float | None: @@ -257,7 +258,7 @@ def baseline(self) -> float | None: @property def unplaced_reason(self) -> UnplacedReason: if any(option.feasible for option in self.options): - return UnplacedReason.CURRENT_UNMEASURED + return UnplacedReason.CURRENT_UNPRICED if self.unpriced else UnplacedReason.CURRENT_UNMEASURED if not self.has_valid_receipt: return UnplacedReason.NO_RECEIPT if not _declares_warm_bound(self.level): @@ -414,14 +415,12 @@ def __init__( self.current_plan = current_plan self.current: dict[str, tuple[PlanIdentity, PlanLane]] = {} self.current_lanes: dict[str, PlanLane] = {} - self.residents: defaultdict[str, list[PlanIdentity]] = defaultdict(list) + self.anchors: dict[str, frozenset[str]] = {} if current_plan is not None: self.current_lanes = {lane.name: lane for lane in current_plan.lanes} self.current = { entry.identity: (entry, self.current_lanes[entry.lane]) for entry in current_plan.identities } - for entry in current_plan.identities: - self.residents[entry.lane].append(entry) self.refusals: list[Refusal] = [] self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} @@ -482,9 +481,18 @@ def keepable(self, identity: str) -> tuple[PlanIdentity, PlanLane] | None: return None return current - def anchors(self, entry: PlanIdentity) -> bool: - level = self.levels.get(entry.identity) - return entry.warmth is Warmth.PINNED and level is not None and level.come_alive_p95_s == 0 + def staying_anchors(self, choices: Iterable[_Choice]) -> dict[str, frozenset[str]]: + anchors: defaultdict[str, set[str]] = defaultdict(set) + for choice in choices: + current = self.current.get(choice.identity) + if current is None or current[0].warmth is not Warmth.PINNED: + continue + if choice.chosen is None or choice.chosen.current: + anchors[current[1].name].add(choice.identity) + return {lane: frozenset(identities) for lane, identities in anchors.items()} + + def floor_paid(self, lane: str, identity: str) -> bool: + return bool(self.anchors.get(lane, frozenset()) - {identity}) def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: current = self.current.get(identity) @@ -494,20 +502,21 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: return tuple( lane for name, lane in sorted(self.current_lanes.items()) - if lane.envelope.floor_min >= PINNED_FLOOR + if (lazy if name == home else self.floor_paid(name, identity)) + and lane.envelope.floor_min >= PINNED_FLOOR and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle - and ( - lazy - if name == home - else any(self.anchors(entry) for entry in self.residents[name] if entry.identity != identity) - ) ) - def resident_hours(self, lane: PlanLane, requests: Sequence[int], rate: float) -> float: + def resident_hours(self, lane: PlanLane, identity: str, requests: Sequence[int], rate: float | None) -> float: capped = self.within_ceiling(lane) + floor = ( + 0.0 if self.floor_paid(lane.name, identity) else capped.envelope.floor_min * self.parameters.hours_per_month + ) + if rate is None: + return floor window = replace(self.window, ceiling=capped.max_containers) - return window.above_floor_hours(requests, rate, capped.envelope.floor_min) + return floor + window.above_floor_hours(requests, rate, capped.envelope.floor_min) def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> float | None: if _refusal_reason(lane.cell, self.prices) is not None: @@ -529,7 +538,7 @@ def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> fl else self.window.warm_container_hours(requests, rate) ) elif entry.warmth is Warmth.LAZY_RESIDENT: - hours = 0.0 if rate is None else self.resident_hours(lane, requests, rate) + hours = self.resident_hours(lane, identity, requests, rate) else: hours = self.window.cold_container_hours(requests, rate) return hourly * hours @@ -543,11 +552,8 @@ def current_gaps( if self.slowest_receipt(identity, lane) is None: refused = (identity, lane.cell.machine_profile) in self.refused_profiles missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) - if entry.warmth in BOOT_WARMTHS: - boot = self.boots.get((identity, lane.cell, lane.container_shape, entry.warmth)) - if boot is None or boot.seconds is None: - reason = RequestReason.UNMEASURED if boot is None else RequestReason.FAILED - missing.append((Measurement.BOOT_EVIDENCE, reason)) + if entry.warmth in BOOT_WARMTHS and (identity, lane.cell, lane.container_shape, entry.warmth) not in self.boots: + missing.append((Measurement.BOOT_EVIDENCE, RequestReason.UNMEASURED)) return tuple( MeasurementRequest( identity=identity, @@ -628,7 +634,7 @@ def choose(self, identity: str) -> _Choice: for host in hosts: if not _runs_on(receipt, host): continue - lazy_cost = hourly * self.resident_hours(host, requests, rate) + lazy_cost = hourly * self.resident_hours(host, identity, requests, rate) load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) if load is None or load.seconds is None: known = load_requests.get(host.name) @@ -664,6 +670,7 @@ def choose(self, identity: str) -> _Choice: feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None held = None + unpriced = False gaps: tuple[MeasurementRequest, ...] = () current = self.keepable(identity) if current is not None and not any(option.current for option in options): @@ -674,6 +681,7 @@ def choose(self, identity: str) -> _Choice: chosen = min(cheaper, key=lambda option: option.rank) if cheaper else None if chosen is None: held = cost + unpriced = cost is None gaps = self.current_gaps(identity, entry, lane, cost) gapped = { (request.cell, request.container_shape, request.warmth) @@ -694,6 +702,7 @@ def choose(self, identity: str) -> _Choice: has_valid_receipt=bool(receipts), held=held, gaps=gaps, + unpriced=unpriced, ) def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: @@ -901,13 +910,13 @@ def kept_blockers( ) -> tuple[Blocker, ...]: level = self.levels[identity] blockers = [] if receipt is None else list(_warm_blockers(level, receipt)) - if entry.warmth in BOOT_WARMTHS and level.come_alive_p95_s is not None: + if entry.warmth in BOOT_WARMTHS: boot = self.boots.get((identity, lane.cell, lane.container_shape, entry.warmth)) if boot is not None and boot.seconds is not None: blocker = _come_alive_blocker(level, boot.seconds) blockers.extend(() if blocker is None else (blocker,)) - elif level.come_alive_p95_s == 0: - blockers.append(Blocker(field="comeAliveP95S", bound=0, measured=None)) + elif boot is not None or level.come_alive_p95_s == 0: + blockers.append(Blocker(field="comeAliveP95S", bound=level.come_alive_p95_s, measured=None)) return tuple(blockers) def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) -> float | None: @@ -927,6 +936,8 @@ def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) - return window.cold_container_hours(requests, rate) def run(self, *, price_savings: bool = True) -> PlannerResult: + never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] + self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) choices = [self.choose(identity) for identity in sorted(self.levels)] placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} kept = { diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 42eea8e31..585f5f8d0 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -32,6 +32,7 @@ class UnplacedReason(StrEnum): WARM_BOUND_MISSED = "warm-bound-missed" WARM_BOUND_UNDECLARED = "warm-bound-undeclared" CURRENT_UNMEASURED = "current-unmeasured" + CURRENT_UNPRICED = "current-unpriced" class ComeAliveBasis(StrEnum): diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 463ad80a8..0141707b1 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -882,6 +882,21 @@ def test_a_kept_lane_on_a_refused_cell_is_kept_and_reported_unpriced(): } +@pytest.mark.parametrize( + "cell", + [Cell(machine_profile="l4", gpu_product="NVIDIA-A10G"), Cell(machine_profile="h100", gpu_product="NVIDIA-H100")], +) +def test_a_current_lane_on_a_refused_cell_is_kept_beside_a_measured_priced_option(cell): + current = single_lane_plan(cell=cell) + result = run(level(come_alive_p95_s=0), [receipt()], prices=L4_ONLY, current=current) + assert result.plan == current + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNPRICED) + assert report.to_dict()["unplacedReason"] == "current-unpriced" + assert result.report.unpriced_lanes == ("shared",) + assert result.report.measurement_requests == () + + def test_a_kept_lane_never_exceeds_the_reviewed_ceiling(): current = single_lane_plan(max_containers=4, envelope=Envelope(floor_min=2, floor_max=3)) result = run(level(come_alive_p95_s=0), [], prices=L4_ONLY, current=current, max_containers=1) @@ -898,6 +913,8 @@ def test_a_kept_lane_never_exceeds_the_reviewed_ceiling(): (120, (), ()), (120, (boot(seconds=200.0),), (Blocker(field="comeAliveP95S", bound=120, measured=200.0),)), (300, (boot(seconds=200.0),), ()), + (120, (boot(outcome=BootOutcome.FAILED),), (Blocker(field="comeAliveP95S", bound=120, measured=None),)), + (None, (boot(outcome=BootOutcome.TIMED_OUT),), (Blocker(field="comeAliveP95S", bound=None, measured=None),)), ], ) def test_a_kept_cold_placement_names_the_come_alive_bound_it_is_known_to_miss(bound, boots, expected): @@ -908,7 +925,8 @@ def test_a_kept_cold_placement_names_the_come_alive_bound_it_is_known_to_miss(bo assert report.blocked_by == expected assert report.monthly_cost == 0 if expected and expected[0].measured is None: - assert report.to_dict()["blockedBy"] == [{"field": "comeAliveP95S", "bound": 0}] + bounded = {} if bound is None else {"bound": bound} + assert report.to_dict()["blockedBy"] == [{"field": "comeAliveP95S", **bounded}] def test_an_identity_without_a_feasible_cell_or_a_current_lane_is_left_out(): @@ -1010,6 +1028,37 @@ def test_a_cold_lane_without_boot_evidence_is_kept_and_asks_for_it(): assert request.cost_bound == pytest.approx((1 / 10.0 + 1 * 2 * 900) / 3600 * SCALE) +@pytest.mark.parametrize("outcome", [BootOutcome.FAILED, BootOutcome.TIMED_OUT]) +def test_a_cold_lane_whose_boot_did_not_succeed_moves_to_a_measured_option(outcome): + current = single_lane_plan(name="cold", warmth=Warmth.COLD_WITH_BOUND) + boots = [boot(seconds=0.0, outcome=outcome)] + result = run( + level(come_alive_p95_s=120), [receipt()], boots, prices=L4_ONLY, hourly={IDENTITY: {0: 1}}, current=current + ) + report = result.report.identity(IDENTITY) + assert (report.status, report.chosen.warmth) == (IdentityStatus.PLACED, Warmth.PINNED) + assert [(entry.lane, entry.warmth) for entry in result.plan.identities] == [("org-model-l4", Warmth.PINNED)] + (request,) = result.report.measurement_requests + assert (request.measure, request.warmth, request.reason) == ( + (Measurement.BOOT_EVIDENCE,), + Warmth.COLD_WITH_BOUND, + RequestReason.FAILED, + ) + assert request.saving_bound == pytest.approx(MONTH - (1 / 10.0 + 1 * 2 * 900) / 3600 * SCALE) + + +def test_a_lazy_resident_whose_load_failed_moves_to_a_measured_option(): + current = plan_of( + [plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("shared", Warmth.LAZY_RESIDENT)} + ) + loads = [boot(warmth=Warmth.LAZY_RESIDENT, outcome=BootOutcome.FAILED, uri="loads/failed.json")] + result = run(resident_levels(), RESIDENT_RECEIPTS, loads, prices=L4_ONLY, current=current) + report = result.report.identity(IDENTITY) + assert (report.status, report.lane, report.chosen.warmth) == (IdentityStatus.PLACED, "org-model-l4", Warmth.PINNED) + requests = [(request.warmth, request.lane, request.reason) for request in result.report.measurement_requests] + assert (Warmth.LAZY_RESIDENT, "shared", RequestReason.FAILED) in requests + + def test_an_unmeasured_current_lane_moves_to_a_measured_option_that_is_cheaper(): current = single_lane_plan(name="cold", warmth=Warmth.COLD_WITH_BOUND, cell=A100) result = run( @@ -1176,6 +1225,62 @@ def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_fro assert third.report.unchanged +def test_a_lazy_resident_never_joins_a_lane_its_never_cold_neighbour_leaves(): + current = plan_of( + [plan_lane("own"), plan_lane("shared", cell=A100)], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + receipts = [receipt(identity=ANCHOR, cell=A100), receipt(identity=ANCHOR), receipt(cell=A100), receipt()] + loads = [boot(cell=A100, warmth=Warmth.LAZY_RESIDENT, seconds=20.0, uri="loads/a100.json")] + result = run(resident_levels(), receipts, loads, current=current) + assert [(entry.identity, entry.lane, entry.warmth) for entry in result.plan.identities] == [ + (ANCHOR, "org-anchor-l4", Warmth.PINNED), + (IDENTITY, "own", Warmth.PINNED), + ] + assert result.report.monthly_cost == pytest.approx(2 * MONTH) + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + again = run(resident_levels(), receipts, loads, current=result.plan) + assert again.report.unchanged + + +def test_a_lazy_resident_whose_lane_keeps_no_never_cold_neighbour_pays_the_lane_floor(): + current = plan_of( + [plan_lane("shared")], {"org/warm": ("shared", Warmth.PINNED), IDENTITY: ("shared", Warmth.LAZY_RESIDENT)} + ) + service_levels = levels( + ("org/warm", ServiceLevel(come_alive_p95_s=120, warm_p99_ms=ANY_P99_MS)), + (IDENTITY, ServiceLevel(come_alive_p95_s=60, warm_p99_ms=ANY_P99_MS)), + ) + receipts = [receipt(identity="org/warm"), receipt()] + boots = [boot(identity="org/warm"), *LOADS, boot(seconds=30.0)] + hourly = {"org/warm": {0: 1}, IDENTITY: {0: 1}} + result = run(service_levels, receipts, boots, prices=L4_ONLY, hourly=hourly, current=current) + cold = (1 / 10.0 + 1 * 2 * 900) / 3600 * SCALE + assert [(entry.identity, entry.lane, entry.warmth) for entry in result.plan.identities] == [ + (IDENTITY, "org-model-l4", Warmth.COLD_WITH_BOUND), + ("org/warm", "org-warm-l4", Warmth.COLD_WITH_BOUND), + ] + assert result.report.monthly_cost == pytest.approx(2 * cold) + report = result.report.identity(IDENTITY) + assert report.chosen.monthly_cost == pytest.approx(cold) + assert report.next_cheaper is None + + +def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_than_the_lane_floor(): + current = plan_of([plan_lane("shared")], {IDENTITY: ("shared", Warmth.LAZY_RESIDENT)}) + result = run(level(come_alive_p95_s=60), [receipt()], LOADS, prices=L4_ONLY, current=current) + assert result.plan == current + assert result.report.unchanged + report = result.report.identity(IDENTITY) + assert (report.status, report.chosen.warmth, report.chosen.lane) == ( + IdentityStatus.PLACED, + Warmth.LAZY_RESIDENT, + "shared", + ) + assert report.chosen.monthly_cost == pytest.approx(MONTH) + assert result.report.monthly_cost == pytest.approx(MONTH) + + @pytest.mark.parametrize( ("shared", "anchor_bound"), [ @@ -1662,7 +1767,7 @@ def capped(lane: PlanLane, ceiling: int) -> PlanLane: return replace(lane, max_containers=ceiling, envelope=envelope) -@pytest.mark.parametrize("seed", range(40)) +@pytest.mark.parametrize("seed", range(200)) def test_random_current_estates_keep_what_no_measured_option_beats(seed): rng = random.Random(seed) # noqa: S311 -- reproducible test data identities = [f"org/model-{index}" for index in range(rng.randint(2, 6))] @@ -1753,6 +1858,13 @@ def test_random_current_estates_keep_what_no_measured_option_beats(seed): if name in before: assert lane == capped(before[name], 4) placements = {entry.identity: entry for entry in result.plan.identities} + never_cold = {entry.identity for entry in service_levels.identities if entry.level.come_alive_p95_s == 0} + anchored = { + entry.lane + for entry in result.plan.identities + if entry.identity in never_cold and residents.get(entry.identity) == (entry.lane, Warmth.PINNED) + } + rates = {price.name: price for price in prices.machine_profiles} for entry in result.report.identities: level = entry.service_level was = residents.get(entry.identity) @@ -1774,6 +1886,11 @@ def test_random_current_estates_keep_what_no_measured_option_beats(seed): if chosen.warmth is Warmth.LAZY_RESIDENT: assert chosen.lane in before assert before[chosen.lane].envelope.floor_min >= 1 + if chosen.lane not in anchored: + lane = after[chosen.lane] + hourly_cost = container_hourly_cost(rates[lane.cell.machine_profile], lane.container_shape, 1.0) + assert was == (chosen.lane, Warmth.LAZY_RESIDENT) + assert chosen.monthly_cost >= hourly_cost * lane.envelope.floor_min * MONTH - 0.01 if entry.next_cheaper is not None: looser = relaxed(service_levels, entry.identity, entry.next_cheaper.blocked_by) again = run(looser, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) @@ -1791,6 +1908,14 @@ def test_random_current_estates_keep_what_no_measured_option_beats(seed): **overrides, ) assert rendered(reordered) == rendered(result) - again = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=result.plan, **overrides) - assert again.plan == result.plan + replanned = result + for _ in range(3): + again = run(service_levels, receipts, boots, prices=prices, hourly=hourly, current=replanned.plan, **overrides) + if again.report.unchanged: + break + lanes = {lane.name for lane in replanned.plan.lanes} + for change in again.report.identity_changes: + assert (change.after.lane in lanes, change.after.warmth) == (True, Warmth.LAZY_RESIDENT) + assert all(change.after is None for change in again.report.lane_changes) + replanned = again assert again.report.unchanged From 116befa72e3a228124a07500ea0413f1553109c4 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 17:16:01 +0100 Subject: [PATCH 12/40] feat(placement): plan with GPU availability as measured evidence Some GPU products take minutes to place a container on, or fail to place one at all. The planner now reads that from boot evidence and bounds plans by the accelerators an environment can obtain. - Boot evidence records placementWaitS, the time an attempt waited for a container, and outcome unplaced for an attempt that never got one. - A cold warmth's come-alive is its placement wait plus its boot, each at the same percentile: the boot over the identity's attempts at the cell, and the placement wait over every cold attempt on the GPU product. From 20 attempts the 95th percentile by nearest rank stands, below that the slowest, and a failed or unplaced attempt ranks above every success. - A product whose cold attempts were more than 5% unplaced is not chosen for an identity whose comeAliveP95S fits the new hold_budget_s parameter, unless a candidate list names a second product where the identity has a receipt meeting its warm bounds. Plan lanes gain fallbackCells for that list, which is priced at its dearest member. - A placement-capacity document states the total, per-product and shared-quota ceilings. plan raises CapacityExceededError, carrying the refused plan and its report, when the envelope maxima exceed one, and the report lists each ceiling's use. - The report's scarcity section names every cheaper option that scarcity alone blocks, so a reviewer sees when it keeps an identity warm. --- packages/sie_placement/README.md | 230 ++++-- .../src/sie_placement/planner.py | 447 ++++++++-- .../sie_placement/src/sie_placement/report.py | 131 ++- .../sie_placement/src/sie_placement/schema.py | 124 ++- .../tests/fixtures/boot-evidence.json | 1 + .../tests/fixtures/capacity.json | 19 + .../sie_placement/tests/fixtures/plan.json | 6 + .../fixtures/planner/expected-report.json | 3 + .../sie_placement/tests/test_availability.py | 781 ++++++++++++++++++ packages/sie_placement/tests/test_planner.py | 22 +- packages/sie_placement/tests/test_schema.py | 132 +++ 11 files changed, 1750 insertions(+), 146 deletions(-) create mode 100644 packages/sie_placement/tests/fixtures/capacity.json create mode 100644 packages/sie_placement/tests/test_availability.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 7a8be6e86..304094fcf 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -29,7 +29,8 @@ name the offending field by its JSON path, for example | `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape | | `placement-price-list` | `PriceList` | Accelerator, core and memory rates per machine profile, and the region multiplier per placement | | `placement-demand` | `Demand` | Hourly request counts per identity and operation over a UTC window | -| `placement-plan` | `Plan` | Lanes with their cell, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | +| `placement-capacity` | `Capacity` | The accelerators a plan may hold: in total, per GPU product, and in a quota it shares with other environments | +| `placement-plan` | `Plan` | Lanes with their cell, fallback cells, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | Each type parses with `from_dict(data)` and serializes with `to_dict()`. Constructing a type in code applies the same rules as parsing it: @@ -74,11 +75,16 @@ A plan gives every identity one warmth: `pinned`, `cold-with-bound`, `lazy-resident` identity sits on a lane whose envelope keeps `floorMin` at 1 or more; every other warmth sits on a lane whose `floorMin` is 0. Boot evidence records `cold-with-bound` boots, `snapshot-cold` restores and -`lazy-resident` loads, with outcome `succeeded`, `failed` or `timed-out`. -`borrowed: true` marks a boot measured in another environment whose -boot-shaping inputs match: the identity and its model revision, the image, the -realized GPU product and count, the container shape, the warmth and the -placement class. Whoever supplies the evidence decides the match. +`lazy-resident` loads, one attempt per document, with outcome `succeeded`, +`failed`, `timed-out` or `unplaced`. `placementWaitS` is the time the attempt +waited for a container to be placed before its `bootSeconds` began. An +`unplaced` attempt never got a container: it records no `bootSeconds`, and +`placementWaitS`, when present, is how long it waited before giving up. A +`lazy-resident` load runs on a container that is already placed, so it records +neither. `borrowed: true` marks an attempt measured in another environment +whose boot-shaping inputs match: the identity and its model revision, the +image, the realized GPU product and count, the container shape, the warmth and +the placement class. Whoever supplies the evidence decides the match. ## Lanes @@ -93,6 +99,27 @@ refers to its lane by that name. Two plan lanes may carry the same three tokens, and so name one chart lane, which cannot give them separate envelopes or concurrency; the planner's report lists every such chart lane. +`fallbackCells` is an optional ordered candidate list: cells, after `cell`, +that the lane's containers may be placed on when `cell` cannot be placed. No +fallback cell repeats `cell` or another fallback cell. A realized card is +billed at its own price, so the planner prices such a lane at its dearest +member. + +## Capacity + +A `placement-capacity` document states the accelerators an environment can +obtain. Each field is optional, and at least one is declared: + +| Field | Meaning | +| --- | --- | +| `total` | The accelerators every lane of the plan may hold together | +| `types` | Per `gpuProduct`, the `accelerators` the lanes that may be placed on that product may hold together | +| `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | + +A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane +with fallback cells counts against the ceiling of every product it names, and +once against `total`. + ## Envelope A plan lane's envelope bounds what an online tuner may change without a new @@ -115,10 +142,17 @@ result = plan( demand, current_plan, PlannerParameters(placement="us-central1", max_containers=4, fallback_floor=2, cooldown_s=900), + capacity=capacity, ) print(render(result.plan.to_dict()), render(result.report.to_dict())) ``` +`capacity` is optional. A plan whose envelope maxima exceed a capacity ceiling +is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` +whose message names each exceeded ceiling and the lanes that hold it, whose +`exceeded` lists those ceilings, and whose `result` carries the refused plan +and its report. + `plan` is a pure function of its documents. The same inputs render to the same bytes in whatever order the receipts, boot evidence, identities and demand series arrive. An identity that stays on its current placement keeps its @@ -136,6 +170,7 @@ may rejoin that lane as a lazy resident on the next run. | `fallback_floor` | The least containers an active zero-floor lane holds, the `sie-cluster` chart's `autoscaling.fallbackReplicas` | | `cooldown_s` | How long an active lane keeps its containers after its last request, the chart's `autoscaling.cooldownPeriod` | | `buffer_containers` | The declared warm buffer above a lane's floor, `0` by default | +| `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | | `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | @@ -154,11 +189,19 @@ document: class, the cell of its current lane, is blocked by the undeclared bound, and an identity without a current lane has no unblocked cell. 3. A cell offers `pinned`, and `cold-with-bound` or `snapshot-cold` where boot - evidence for the identity, cell and container shape exists. Below 20 - successful boots the slowest stands for the come-alive percentile; from 20, - their 95th percentile by nearest rank does, so one slow boot stops deciding - once enough boots exist. A boot that did not succeed withdraws the warmth, - and the environment's own evidence replaces borrowed evidence. A + evidence for the identity, cell and container shape exists. A cold + warmth's come-alive is its placement wait plus its boot, each taken at the + same percentile. The boot is taken over the identity's attempts at that + cell, container shape and warmth that got a container. The placement wait + is taken over every cold attempt on the cell's GPU product, whatever the + identity, that records `placementWaitS` or was `unplaced`; a product + without such an attempt adds no wait. Below 20 attempts the slowest stands + for the percentile; from 20, their 95th percentile by nearest rank does, so + one slow attempt stops deciding once enough exist. An attempt that did not + succeed, a failed or timed-out boot or an unplaced placement, ranks above + every success, so the warmth is withdrawn when the percentile falls on one. + The environment's own attempts replace borrowed ones, per cell, container + shape and warmth for the boot and per GPU product for the placement wait. A `comeAliveP95S` of `0` admits `pinned` only. 4. A cell also offers `lazy-resident` on each host lane with its cell, container shape and concurrency. A host lane is a lane of the current plan @@ -169,31 +212,43 @@ document: so no other choice moves them. The identity's own current lane is also a host when it is a lazy resident there already. The come-alive is the identity's `lazy-resident` boot evidence, its measured load onto that cell - and container shape, under the same statistic. The option costs what the + and container shape, under the same statistic, with no placement wait, + since the host's container is already placed. The option costs what the identity's demand needs above the host lane's floor, plus the floor itself on a lane where no such identity stays pinned, since no other identity pays - it. -5. The cheapest unblocked option wins. Ties go to the current placement, - then to the faster come-alive. + it. A lane with fallback cells hosts no lazy resident. +5. A GPU product is unreliable when more than 5% of the cold attempts on it + were `unplaced`, counting the environment's own attempts in place of + borrowed ones. For an identity whose `comeAliveP95S` is at most + `hold_budget_s`, every option on an unreliable product is blocked by + `placementFailureRate`. Such an identity may still be pinned on an + unreliable product through a candidate list: a lane whose fallback cell is + on another product no cheaper than the first, where the identity has a + receipt meeting its warm bounds at the same container shape and + concurrency on both. The list is priced at the dearer product and at the + slower of the two receipts. +6. The cheapest unblocked option wins. Ties go to the current placement, + then to the faster come-alive, then to a lane without fallback cells. The current placement is the identity's current lane when that lane has the -option's cell, container shape and concurrency, the identity's warmth is the -option's, and the lane's `queuePool` and `bundle` are the ones the parameters -give the identity; a `lazy-resident` option must also be hosted by that lane. -A current placement that no option matches lacks evidence: a receipt at its -lane's cell, container shape and concurrency, a boot for a cold warmth, or a -load for a lazy resident. Unless it is known to miss a bound, through a receipt -at the lane that misses a warm bound, a measured come-alive over the bound, a -boot or load at the lane that did not succeed, or a cold or `lazy-resident` -warmth under a bound of `0`, the identity keeps that placement and leaves it -only for an unblocked option at least a cent cheaper than the placement. The -placement is priced as an option of its warmth at the lane's fastest receipt, -or without one at its floor alone: one container all month for `pinned`, for -`lazy-resident` nothing beyond the lane's floor, or the floor itself where no -identity whose `comeAliveP95S` is `0` stays pinned on the lane, and the burst -tails for a cold warmth. What the placement lacks becomes a measurement -request. Such a placement on a lane the price list cannot price has no cost to -compare, so the identity keeps it. +option's cell and fallback cells, container shape and concurrency, the +identity's warmth is the option's, and the lane's `queuePool` and `bundle` are +the ones the parameters give the identity; a `lazy-resident` option must also be +hosted by that lane. A current placement that no option matches lacks evidence: +a receipt at its lane's cell, container shape and concurrency, a boot for a cold +warmth, or a load for a lazy resident. Unless it is known to miss a bound, +through a receipt at the lane that misses a warm bound, a measured come-alive +over the bound, a boot or load at the lane that did not succeed, a cold or +`lazy-resident` warmth under a bound of `0`, or an unreliable product without a +fallback cell on another product under a bound `hold_budget_s` fits, the +identity keeps that placement and leaves it only for an unblocked option at +least a cent cheaper than the placement. The placement is priced as an option of +its warmth at the lane's fastest receipt, or without one at its floor alone: one +container all month for `pinned`, for `lazy-resident` nothing beyond the lane's +floor, or the floor itself where no identity whose `comeAliveP95S` is `0` stays +pinned on the lane, and the burst tails for a cold warmth. What the placement +lacks becomes a measurement request. Such a placement on a lane the price list +cannot price has no cost to compare, so the identity keeps it. An identity that stays on its current placement keeps that lane exactly as it is, beside whoever else the lane hosts: its cell, container shape, @@ -225,7 +280,9 @@ resident's demand needs beyond the floor: on a lane with a floor, the containers an hour needs above it, and on a zero-floor lane, the work and the tails a cold warmth pays. A placed identity's demand is priced at its chosen receipt, and a kept identity's at the slowest receipt it has for its lane's -cell, container shape and concurrency, under that lane's own ceiling. +cells, container shape and concurrency, under that lane's own ceiling. A lane +with fallback cells costs the hourly rate of its dearest cell, and a candidate +list's demand is priced at the slower of its two receipts. ### Envelope @@ -235,7 +292,10 @@ the ceiling. A floor raised on a zero-floor lane realizes as at least `fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least `fallback_floor` capped by the ceiling. The envelope carries the floor range only. A lane carried from the current plan keeps its envelope, capped by the -ceiling. +ceiling. Given a `capacity` document, the envelope maxima of every lane must +fit each of its ceilings, or the plan is refused as described above. A +loosened service level whose plan the ceilings would refuse saves nothing in +the report. ### Report @@ -248,26 +308,36 @@ The report renders as a `placement-report` document: effective service level and demand, its lane and `monthlyCost`, what its demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or - `borrowed-boot-evidence`) and, for a lazy resident, its host `lane`, the peak - containers its demand needs, and `nextCheaper`: the nearest option cheaper - than the choice, or than a kept identity's current placement, that a - service level blocks, the bounds that block it, and `saving`, the fall in - the plan total found by planning again from this plan with each of those - bounds loosened to what the option measured, rounded up to a whole second or - millisecond and never below 1. A bound the identity does not declare blocks - with no `bound`, and loosening it declares it. The loosened bounds may admit - a cheaper option still, and the saving counts it. The saving is 0 when the - total would not fall, as for an identity whose lane keeps its floor for its - other residents. An identity without a placement names its - `unplacedReason`: `no-receipt`, `warm-bound-missed`, - `warm-bound-undeclared` when no cell is on its current class, - `current-unmeasured` when it keeps a current placement that lacks evidence - and no unblocked option is cheaper, or `current-unpriced` when it keeps a - current placement on a lane the price list cannot price. A kept identity's - `blockedBy` names the bounds its kept placement is known to miss: its - receipt's warm latencies, a measured come-alive over the bound, and with no - `measured` value a boot or load at its lane that did not succeed, or a cold - or `lazy-resident` warmth under a bound of `0`, which needs no measurement. + `borrowed-boot-evidence`), the `placementWaitS` part of a cold come-alive + where a placement wait was measured, the `fallbackCells` of a candidate list + and, for a lazy resident, its host `lane`, the peak containers its demand + needs, and `nextCheaper`: the nearest option cheaper than the choice, or than + a kept identity's current placement, that a service level blocks, the bounds + that block it, and `saving`, the fall in the plan total found by planning + again from this plan with each of those bounds loosened to what the option + measured, rounded up to a whole second or millisecond and never below 1. A + bound the identity does not declare blocks with no `bound`, and loosening it + declares it. A `placementFailureRate` blocker, whose `bound` is `0.05` and + `measured` the product's failure rate, is loosened by raising `comeAliveP95S` + to one second above `hold_budget_s`. The loosened bounds may admit a cheaper + option still, and the saving counts it. The saving is 0 when the total would + not fall, as for an identity whose lane keeps its floor for its other + residents. An identity without a placement names its `unplacedReason`: + `no-receipt`, `warm-bound-missed`, `warm-bound-undeclared` when no cell is on + its current class, `placement-unreliable` when its options are on unreliable + products only, `current-unmeasured` when it keeps a current placement that + lacks evidence and no unblocked option is cheaper, or `current-unpriced` when + it keeps a current placement on a lane the price list cannot price. A kept + identity's `blockedBy` names the bounds its kept placement is known to miss: + its receipt's warm latencies, a measured come-alive over the bound, an + unreliable product, and with no `measured` value a boot or load at its lane + that did not succeed, or a cold or `lazy-resident` warmth under a bound of + `0`, which needs no measurement. +- `capacity`: for each ceiling of the `capacity` document, its `scope` + (`total`, `type` with its `gpuProduct`, or `shared-quota` with + `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope + maxima hold against it, the `lanes` that hold them, and whether it is + `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, @@ -278,28 +348,42 @@ The report renders as a `placement-report` document: - `measurementRequests`: options that lack evidence, ranked by the saving they could unlock. A profile without a receipt is bounded by price alone: one accelerator at its rate, kept warm or, where the come-alive bound allows, - paying burst tails only. A cell whose receipt has no successful boot asks for - boot evidence and is bounded by its cold cost, and a host lane without the - identity's load asks for `lazy-resident` boot evidence on it, bounded by + paying burst tails only. A profile on an unreliable product is not requested + for an identity whose bound `hold_budget_s` fits. A cell whose receipt has no + successful boot asks for boot evidence and is bounded by its cold cost, with + reason `failed`, or `unplaced` when no attempt got a container or the + product's placement wait falls on an unplaced attempt, and a host lane without + the identity's load asks for `lazy-resident` boot evidence on it, bounded by what that option would cost. Savings are measured against the choice, or against a kept identity's current placement. A request that cannot save is - dropped, and the requests of an identity without a placement or a current - one carry no bound and rank first. The evidence a kept current placement - lacks is requested at its lane, named in `lane` with its `concurrency`, and - also ranks first without a bound; a profile it asks for there is not asked - for again. -- `comeAliveLevers`: cold and `lazy-resident` warmths whose measured - come-alive misses the bound on a cell that meets the warm bounds for less - than the choice, or than a kept identity's current placement, listed only - when loosening the identity's bound to `permittedBoundS`, the bound the - measured come-alive would permit, lowers the plan total. Both savings are - found by planning again from this plan and taking the fall in its total: + dropped, and the requests of an identity without a placement or a current one + carry no bound and rank first. The evidence a kept current placement lacks is + requested at its lane, named in `lane` with its `concurrency`, and also ranks + first without a bound; a profile it asks for there is not asked for again. +- `comeAliveLevers`: cold and `lazy-resident` warmths whose measured come-alive + misses the bound on a cell that meets the warm bounds for less than the + choice, or than a kept identity's current placement, listed only when + loosening the identity's bound to `permittedBoundS`, the bound the measured + come-alive would permit, lowers the plan total. Both savings are found by + planning again from this plan and taking the fall in its total: `savingAtPermittedBound` with the bound loosened, and `savingUnderBound` with - the warmth's boot measured at the identity's bound, which is 0 under a bound - of 0, since no boot comes under it, and when the total would not fall. An - identity that shares its lane saves nothing of the lane's floor by leaving, - since the other residents keep it. Levers rank by the saving under the bound, - then by the saving at the permitted bound. + the warmth's boot measured at the identity's bound less its placement wait, + which is 0 under a bound of 0, since no boot comes under it, when the + placement wait alone exceeds the bound, and when the total would not fall. A + lever names the `placementWaitS` part of its come-alive where one was + measured. An identity that shares its lane saves nothing of the lane's floor + by leaving, since the other residents keep it. Levers rank by the saving under + the bound, then by the saving at the permitted bound. +- `scarcity`: every option cheaper than an identity's choice, or than a kept + identity's current placement, that scarcity alone blocks, so scarcity is + part of why the identity stays warm or pays more: `placement-wait`, a cold + warmth whose boot fits the come-alive bound and whose boot plus placement + wait does not; `unplaced`, a cold warmth whose boot fits the bound and whose + product's placement wait falls on an unplaced attempt; and + `unreliable-type`, an option on an unreliable product under a bound + `hold_budget_s` fits. Each names the option's cell, container shape, warmth + and monthly cost, the bound, the boot and placement wait it measured, and + the product's `placementFailureRate`. - `queueWaitUnattained`: every identity that declares `queueWaitP99Ms`. No plan claims that bound as met. - `warmBoundUndeclared`: every identity that has a cell and declares neither diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index b3e82d23b..bf286ef09 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -13,6 +13,8 @@ from sie_placement.report import ( Blocker, + CapacityScope, + CapacityUse, ComeAliveBasis, IdentityChange, IdentityReport, @@ -27,6 +29,8 @@ RefusedRecord, Report, RequestReason, + Scarcity, + ScarcityReason, SharedChartLane, UnplacedReason, ) @@ -35,6 +39,7 @@ WARM_FLOOR_WARMTHS, BootEvidence, BootOutcome, + Capacity, Cell, ContainerShape, Demand, @@ -58,6 +63,8 @@ PINNED_FLOOR = 1 BOOT_PERCENTILE = 95 BOOT_PERCENTILE_MIN_ROWS = 20 +PLACEMENT_FAILURE_RATE_LIMIT = 0.05 +PLACEMENT_FAILURE_FIELD = "placementFailureRate" COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") @@ -90,6 +97,7 @@ class PlannerParameters: fallback_floor: int cooldown_s: int buffer_containers: int = 0 + hold_budget_s: int = 0 hours_per_month: float = HOURS_PER_MONTH queue_pool: str = "default" bundle: str = "default" @@ -101,6 +109,7 @@ def __post_init__(self) -> None: _require_minimum("fallbackFloor", self.fallback_floor, 0) _require_minimum("cooldownS", self.cooldown_s, 0) _require_minimum("bufferContainers", self.buffer_containers, 0) + _require_minimum("holdBudgetS", self.hold_budget_s, 0) if not (math.isfinite(self.hours_per_month) and self.hours_per_month > 0): raise SchemaError("hoursPerMonth", f"must be greater than 0, got {self.hours_per_month!r}") _require_name("queuePool", self.queue_pool) @@ -123,6 +132,7 @@ def to_dict(self) -> dict[str, object]: "fallbackFloor": self.fallback_floor, "cooldownS": self.cooldown_s, "bufferContainers": self.buffer_containers, + "holdBudgetS": self.hold_budget_s, "hoursPerMonth": self.hours_per_month, "queuePool": self.queue_pool, "bundle": self.bundle, @@ -138,6 +148,13 @@ class PlannerResult: report: Report +class CapacityExceededError(PlannerInputError): + def __init__(self, result: PlannerResult) -> None: + self.result = result + self.exceeded = tuple(use for use in result.report.capacity if use.exceeded) + super().__init__("; ".join(use.describe() for use in self.exceeded)) + + def container_hourly_cost(price: MachineProfilePrice, shape: ContainerShape, region_multiplier: float) -> float: return region_multiplier * ( price.accelerator_per_hour * shape.gpu_count @@ -189,14 +206,46 @@ def cold_container_hours(self, requests: Sequence[int], requests_per_second: flo return seconds / SECONDS_PER_HOUR * self.scale -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, kw_only=True) +class _Placement: + wait_s: float | None + measured: bool + failure_rate: float + borrowed: bool + + +_UNMEASURED_PLACEMENT = _Placement(wait_s=0.0, measured=False, failure_rate=0.0, borrowed=False) + + +@dataclass(frozen=True, slots=True, kw_only=True) class _Boot: - seconds: float | None + boot_s: float | None borrowed: bool + placement: _Placement = _UNMEASURED_PLACEMENT + booted: bool = True + + @property + def seconds(self) -> float | None: + if self.boot_s is None or self.placement.wait_s is None: + return None + if not self.placement.measured: + return self.boot_s + return round(self.boot_s + self.placement.wait_s, 3) + + @property + def placement_wait_s(self) -> float | None: + return self.placement.wait_s if self.placement.measured else None @property def basis(self) -> ComeAliveBasis: - return ComeAliveBasis.BORROWED_BOOT_EVIDENCE if self.borrowed else ComeAliveBasis.BOOT_EVIDENCE + borrowed = self.borrowed or (self.placement.measured and self.placement.borrowed) + return ComeAliveBasis.BORROWED_BOOT_EVIDENCE if borrowed else ComeAliveBasis.BOOT_EVIDENCE + + @property + def reason(self) -> RequestReason: + if not self.booted or (self.boot_s is not None and self.placement.wait_s is None): + return RequestReason.UNPLACED + return RequestReason.FAILED type _BootKey = tuple[str, Cell, ContainerShape, Warmth] @@ -213,14 +262,33 @@ class _Option: blockers: tuple[Blocker, ...] current: bool host: str | None = None + fallbacks: tuple[Receipt, ...] = () + boot_s: float | None = None + placement_wait_s: float | None = None @property def feasible(self) -> bool: return not self.blockers @property - def rank(self) -> tuple[float, bool, float, str, str, str]: - return (self.cost, not self.current, self.come_alive_s, self.order, self.warmth.value, self.host or "") + def cells(self) -> tuple[Cell, ...]: + return (self.receipt.cell, *(receipt.cell for receipt in self.fallbacks)) + + @property + def pricing_receipt(self) -> Receipt: + return min((self.receipt, *self.fallbacks), key=lambda receipt: receipt.throughput.requests_per_second) + + @property + def rank(self) -> tuple[float, bool, float, int, str, str, str]: + return ( + self.cost, + not self.current, + self.come_alive_s, + len(self.fallbacks), + self.order, + self.warmth.value, + self.host or "", + ) def summary(self, saving: float | None = None) -> Option: return Option( @@ -235,6 +303,8 @@ def summary(self, saving: float | None = None) -> Option: blocked_by=self.blockers, saving=saving, lane=self.host, + fallback_cells=self.cells[1:], + placement_wait_s=self.placement_wait_s, ) @@ -250,6 +320,7 @@ class _Choice: held: float | None = None gaps: tuple[MeasurementRequest, ...] = () unpriced: bool = False + withdrawn: tuple[_Option, ...] = () @property def baseline(self) -> float | None: @@ -263,6 +334,8 @@ def unplaced_reason(self) -> UnplacedReason: return UnplacedReason.NO_RECEIPT if not _declares_warm_bound(self.level): return UnplacedReason.WARM_BOUND_UNDECLARED + if any(_fields(option.blockers) == [PLACEMENT_FAILURE_FIELD] for option in self.options): + return UnplacedReason.PLACEMENT_UNRELIABLE return UnplacedReason.WARM_BOUND_MISSED @@ -290,6 +363,14 @@ def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> st return None +def _lane_refusal(lane: PlanLane, prices: Mapping[str, MachineProfilePrice]) -> str | None: + return next((reason for cell in lane.cells if (reason := _refusal_reason(cell, prices)) is not None), None) + + +def _fields(blockers: Iterable[Blocker]) -> list[str]: + return [blocker.field for blocker in blockers] + + def _incomparable_reason(level: ServiceLevel, receipt: Receipt) -> str | None: generate = receipt.request_shape.operation is Operation.GENERATE if level.is_generation and not generate: @@ -334,25 +415,65 @@ def _come_alive_blocker(level: ServiceLevel, seconds: float) -> Blocker | None: return Blocker(field="comeAliveP95S", bound=bound, measured=seconds) -def _come_alive(seconds: Iterable[float]) -> float: - ordered = sorted(float(value) for value in seconds) - if len(ordered) < BOOT_PERCENTILE_MIN_ROWS: - return ordered[-1] - return ordered[(len(ordered) * BOOT_PERCENTILE + 99) // 100 - 1] +def _percentile(samples: Sequence[float | None]) -> float | None: + succeeded = sorted(value for value in samples if value is not None) + if len(samples) < BOOT_PERCENTILE_MIN_ROWS: + return succeeded[-1] if len(succeeded) == len(samples) else None + rank = (len(samples) * BOOT_PERCENTILE + 99) // 100 + return succeeded[rank - 1] if rank <= len(succeeded) else None + + +def _boot_sample(record: BootEvidence) -> float | None: + if record.outcome is not BootOutcome.SUCCEEDED or record.boot_seconds is None: + return None + return float(record.boot_seconds) -def _boot_index(records: Iterable[BootEvidence]) -> dict[_BootKey, _Boot]: +def _placement_index(records: Iterable[BootEvidence]) -> dict[str, _Placement]: + grouped: defaultdict[str, list[BootEvidence]] = defaultdict(list) + for record in records: + if record.warmth in COLD_WARMTHS: + grouped[record.cell.gpu_product].append(record) + index: dict[str, _Placement] = {} + for product, group in grouped.items(): + own = [record for record in group if not record.borrowed] + used = own or group + samples = [ + None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) + for record in used + if record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None + ] + unplaced = sum(record.outcome is BootOutcome.UNPLACED for record in used) + index[product] = _Placement( + wait_s=_percentile(samples) if samples else 0.0, + measured=bool(samples), + failure_rate=unplaced / len(used), + borrowed=not own, + ) + return index + + +def _boot_index(records: Iterable[BootEvidence], placements: Mapping[str, _Placement]) -> dict[_BootKey, _Boot]: grouped: defaultdict[_BootKey, list[BootEvidence]] = defaultdict(list) for record in records: grouped[(record.identity, record.cell, record.container_shape, record.warmth)].append(record) index: dict[_BootKey, _Boot] = {} for key, group in grouped.items(): - own = [record for record in group if not record.borrowed] - used = own or group - if any(record.outcome is not BootOutcome.SUCCEEDED for record in used): - index[key] = _Boot(seconds=None, borrowed=not own) - else: - index[key] = _Boot(seconds=_come_alive(record.boot_seconds for record in used), borrowed=not own) + _, cell, _, warmth = key + placement = placements.get(cell.gpu_product, _UNMEASURED_PLACEMENT) + if warmth not in COLD_WARMTHS: + placement = _UNMEASURED_PLACEMENT + booted = [record for record in group if record.outcome is not BootOutcome.UNPLACED] + own = [record for record in booted if not record.borrowed] + used = own or booted + if not used: + index[key] = _Boot( + boot_s=None, borrowed=all(record.borrowed for record in group), placement=placement, booted=False + ) + continue + index[key] = _Boot( + boot_s=_percentile([_boot_sample(record) for record in used]), borrowed=not own, placement=placement + ) return index @@ -368,7 +489,7 @@ def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: def _runs_on(receipt: Receipt, lane: PlanLane) -> bool: return ( - lane.cell == receipt.cell + receipt.cell in lane.cells and lane.container_shape == receipt.container_shape and lane.concurrency == receipt.concurrency ) @@ -384,6 +505,7 @@ def __init__( demand: Demand, current_plan: Plan | None, parameters: PlannerParameters, + capacity: Capacity | None = None, ) -> None: receipts = tuple(receipts) boot_evidence = tuple(boot_evidence) @@ -396,6 +518,7 @@ def __init__( f"the current plan is for placement {current_plan.placement!r}, not {parameters.placement!r}" ) self.parameters = parameters + self.capacity = capacity self.service_levels = service_levels self.receipt_records = receipts self.boot_records = boot_evidence @@ -425,7 +548,10 @@ def __init__( self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} self.receipts = self._accept_receipts(receipts) - self.boots = _boot_index(self._accept_boot_evidence(boot_evidence)) + self.placements = _placement_index( + record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None + ) + self.boots = _boot_index(self._accept_boot_evidence(boot_evidence), self.placements) def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[str, Receipt]]]: accepted: defaultdict[str, list[tuple[str, Receipt]]] = defaultdict(list) @@ -456,7 +582,7 @@ def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvi def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) - def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: + def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: tuple[Cell, ...] = ()) -> bool: current = self.current.get(identity) if current is None: return False @@ -465,9 +591,45 @@ def is_current(self, identity: str, receipt: Receipt, warmth: Warmth) -> bool: entry.warmth is warmth and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) + and lane.cells == (receipt.cell, *fallbacks) and _runs_on(receipt, lane) ) + def hourly(self, receipt: Receipt) -> float: + return container_hourly_cost( + self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier + ) + + def lane_hourly(self, lane: PlanLane) -> float: + return max( + container_hourly_cost(self.prices[cell.machine_profile], lane.container_shape, self.multiplier) + for cell in lane.cells + ) + + def tight(self, level: ServiceLevel) -> bool: + bound = level.come_alive_p95_s + return bound is not None and bound <= self.parameters.hold_budget_s + + def unreliable(self, gpu_product: str) -> bool: + placement = self.placements.get(gpu_product) + return placement is not None and placement.failure_rate > PLACEMENT_FAILURE_RATE_LIMIT + + def reliability_blockers(self, level: ServiceLevel, cells: Sequence[Cell]) -> tuple[Blocker, ...]: + first = cells[0].gpu_product + if not self.tight(level) or not self.unreliable(first): + return () + if any(cell.gpu_product != first for cell in cells[1:]): + return () + rate = self.placements[first].failure_rate + return (Blocker(field=PLACEMENT_FAILURE_FIELD, bound=PLACEMENT_FAILURE_RATE_LIMIT, measured=rate),) + + @staticmethod + def warm_blockers(level: ServiceLevel, receipt: Receipt, current_class: Cell | None) -> tuple[Blocker, ...]: + warm = _warm_blockers(level, receipt) + if receipt.cell != current_class: + warm = (*warm, *_undeclared_blockers(level, receipt)) + return warm + def current_class(self, identity: str) -> Cell | None: current = self.current.get(identity) return None if current is None else current[1].cell @@ -504,6 +666,7 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: for name, lane in sorted(self.current_lanes.items()) if (lazy if name == home else self.floor_paid(name, identity)) and lane.envelope.floor_min >= PINNED_FLOOR + and not lane.fallback_cells and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -519,9 +682,9 @@ def resident_hours(self, lane: PlanLane, identity: str, requests: Sequence[int], return floor + window.above_floor_hours(requests, rate, capped.envelope.floor_min) def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> float | None: - if _refusal_reason(lane.cell, self.prices) is not None: + if _lane_refusal(lane, self.prices) is not None: return None - hourly = container_hourly_cost(self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier) + hourly = self.lane_hourly(lane) requests = self.requests.get(identity, ()) rate = max( ( @@ -574,19 +737,19 @@ def choose(self, identity: str) -> _Choice: level = self.levels[identity] requests = self.requests.get(identity, ()) options: list[_Option] = [] + withdrawn: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} load_requests: dict[str, MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) hosts = self.lazy_hosts(identity) for order, receipt in receipts: - hourly = container_hourly_cost( - self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier - ) + hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second - warm = _warm_blockers(level, receipt) - if receipt.cell != current_class: - warm = (*warm, *_undeclared_blockers(level, receipt)) + blocked = ( + *self.warm_blockers(level, receipt, current_class), + *self.reliability_blockers(level, (receipt.cell,)), + ) options.append( _Option( receipt=receipt, @@ -595,7 +758,7 @@ def choose(self, identity: str) -> _Choice: cost=hourly * self.window.warm_container_hours(requests, rate), come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, - blockers=warm, + blockers=blocked, current=self.is_current(identity, receipt, Warmth.PINNED), ) ) @@ -603,7 +766,7 @@ def choose(self, identity: str) -> _Choice: for warmth in COLD_WARMTHS: boot = self.boots.get((identity, receipt.cell, receipt.container_shape, warmth)) if boot is None or boot.seconds is None: - if warmth is Warmth.COLD_WITH_BOUND and not warm and level.come_alive_p95_s != 0: + if warmth is Warmth.COLD_WITH_BOUND and not blocked and level.come_alive_p95_s != 0: key = (receipt.cell, receipt.container_shape) known = boot_requests.get(key) if known is None or cold_cost < known.cost_bound: @@ -613,10 +776,24 @@ def choose(self, identity: str) -> _Choice: container_shape=receipt.container_shape, measure=(Measurement.BOOT_EVIDENCE,), warmth=warmth, - reason=RequestReason.UNMEASURED if boot is None else RequestReason.FAILED, + reason=RequestReason.UNMEASURED if boot is None else boot.reason, cost_bound=cold_cost, saving_bound=None, ) + if boot is not None and boot.boot_s is not None and not blocked: + withdrawn.append( + _Option( + receipt=receipt, + order=order, + warmth=warmth, + cost=cold_cost, + come_alive_s=math.inf, + basis=boot.basis, + blockers=(Blocker(field="comeAliveP95S", bound=level.come_alive_p95_s, measured=None),), + current=False, + boot_s=boot.boot_s, + ) + ) continue blocker = _come_alive_blocker(level, boot.seconds) options.append( @@ -627,8 +804,10 @@ def choose(self, identity: str) -> _Choice: cost=cold_cost, come_alive_s=boot.seconds, basis=boot.basis, - blockers=warm if blocker is None else (*warm, blocker), + blockers=blocked if blocker is None else (*blocked, blocker), current=self.is_current(identity, receipt, warmth), + boot_s=boot.boot_s, + placement_wait_s=boot.placement_wait_s, ) ) for host in hosts: @@ -638,7 +817,7 @@ def choose(self, identity: str) -> _Choice: load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) if load is None or load.seconds is None: known = load_requests.get(host.name) - if not warm and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): + if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): load_requests[host.name] = MeasurementRequest( identity=identity, cell=host.cell, @@ -661,12 +840,15 @@ def choose(self, identity: str) -> _Choice: cost=lazy_cost, come_alive_s=load.seconds, basis=load.basis, - blockers=warm if blocker is None else (*warm, blocker), + blockers=blocked if blocker is None else (*blocked, blocker), current=self.is_current(identity, receipt, Warmth.LAZY_RESIDENT) and self.current[identity][1].name == host.name, host=host.name, + boot_s=load.boot_s, ) ) + if self.tight(level): + options.extend(self.candidate_lists(identity, level, receipts, requests, current_class)) feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None held = None @@ -703,17 +885,57 @@ def choose(self, identity: str) -> _Choice: held=held, gaps=gaps, unpriced=unpriced, + withdrawn=tuple(withdrawn), ) + def candidate_lists( + self, + identity: str, + level: ServiceLevel, + receipts: Sequence[tuple[str, Receipt]], + requests: Sequence[int], + current_class: Cell | None, + ) -> list[_Option]: + lists = [] + for order, receipt in receipts: + if not self.unreliable(receipt.cell.gpu_product) or self.warm_blockers(level, receipt, current_class): + continue + for second_order, second in receipts: + if ( + second.cell.gpu_product == receipt.cell.gpu_product + or self.hourly(second) < self.hourly(receipt) + or second.container_shape != receipt.container_shape + or second.concurrency != receipt.concurrency + or self.warm_blockers(level, second, current_class) + ): + continue + hourly = max(self.hourly(receipt), self.hourly(second)) + rate = min(receipt.throughput.requests_per_second, second.throughput.requests_per_second) + lists.append( + _Option( + receipt=receipt, + fallbacks=(second,), + order=f"{order}\n{second_order}", + warmth=Warmth.PINNED, + cost=hourly * self.window.warm_container_hours(requests, rate), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=(), + current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,)), + ) + ) + return lists + def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} measured.update( request.cell.machine_profile for request in choice.gaps if Measurement.RECEIPT in request.measure ) bound = choice.level.come_alive_p95_s + tight = self.tight(choice.level) requests = [] for profile in sorted(self.price_list.machine_profiles, key=lambda price: price.name): - if profile.name in measured: + if profile.name in measured or (tight and self.unreliable(profile.gpu_product)): continue accelerator = profile.accelerator_per_hour * self.multiplier candidates = [(accelerator * self.window.warm_container_hours((), 1.0), Warmth.PINNED)] @@ -770,8 +992,12 @@ def replanned_total( self.demand, current, self.parameters, + self.capacity, ) - return planner.run(price_savings=False).report.monthly_cost + report = planner.run(price_savings=False).report + if any(use.exceeded for use in report.capacity): + return math.inf + return report.monthly_cost def loosened(self, identity: str, bounds: Mapping[str, int]) -> ServiceLevels: fields = {_LEVEL_FIELDS[name]: bound for name, bound in bounds.items()} @@ -790,11 +1016,25 @@ def saving_when_loosened(self, identity: str, bounds: Mapping[str, int], written self.savings[key] = _saving(total, replanned) return self.savings[key] + def loosening(self, blockers: Iterable[Blocker]) -> dict[str, int]: + bounds: dict[str, int] = {} + for blocker in blockers: + if blocker.field == PLACEMENT_FAILURE_FIELD: + name, bound = "comeAliveP95S", self.parameters.hold_budget_s + 1 + elif blocker.measured is not None: + name, bound = blocker.field, _permitted(blocker.measured) + else: + continue + bounds[name] = max(bound, bounds.get(name, 0)) + return bounds + def booted_within(self, identity: str, option: _Option, bound: int) -> tuple[BootEvidence, ...]: key = (identity, option.receipt.cell, option.receipt.container_shape, option.warmth) + budget = max(0.0, bound - (option.placement_wait_s or 0.0)) return tuple( - replace(record, boot_seconds=min(record.boot_seconds, bound)) + replace(record, boot_seconds=min(record.boot_seconds, budget)) if (record.identity, record.cell, record.container_shape, record.warmth) == key + and record.boot_seconds is not None else record for record in self.boot_records ) @@ -839,6 +1079,7 @@ def levers(self, choice: _Choice, written: Plan, total: float) -> list[Lever]: saving_under_bound=under, permitted_bound_s=permitted, saving_at_permitted_bound=saving, + placement_wait_s=option.placement_wait_s, ) ) return levers @@ -851,15 +1092,91 @@ def next_cheaper(self, choice: _Choice, baseline: float | None, written: Plan, t return None cost = max(option.cost for option in cheaper) nearest = min((option for option in cheaper if option.cost == cost), key=lambda option: option.rank) - bounds = { - blocker.field: _permitted(blocker.measured) for blocker in nearest.blockers if blocker.measured is not None - } + bounds = self.loosening(nearest.blockers) return nearest.summary(saving=self.saving_when_loosened(choice.identity, bounds, written, total)) + def scarcity(self, choice: _Choice) -> list[Scarcity]: + baseline = choice.baseline + if baseline is None: + return [] + bound = choice.level.come_alive_p95_s + candidates = [(option, _scarcity_reason(option, bound)) for option in choice.options] + candidates.extend( + (option, ScarcityReason.UNPLACED) + for option in choice.withdrawn + if bound is None or (bound > 0 and (option.boot_s or 0.0) <= bound) + ) + best: dict[tuple[Cell, ContainerShape, Warmth, str], tuple[_Option, ScarcityReason]] = {} + for option, reason in candidates: + if reason is None or round(baseline - option.cost, 2) <= 0: + continue + key = (option.receipt.cell, option.receipt.container_shape, option.warmth, option.host or "") + known = best.get(key) + if known is None or option.rank < known[0].rank: + best[key] = (option, reason) + return [ + Scarcity( + identity=choice.identity, + reason=reason, + cell=option.receipt.cell, + container_shape=option.receipt.container_shape, + warmth=option.warmth, + monthly_cost=option.cost, + placement_failure_rate=self.placements.get( + option.receipt.cell.gpu_product, _UNMEASURED_PLACEMENT + ).failure_rate, + come_alive_bound_s=bound, + boot_s=option.boot_s, + placement_wait_s=option.placement_wait_s, + lane=option.host, + ) + for option, reason in best.values() + ] + + def capacity_use(self, plan: Plan) -> tuple[CapacityUse, ...]: + capacity = self.capacity + if capacity is None: + return () + + def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: + counted = [lane for lane in lanes if lane.envelope.floor_max] + accelerators = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in counted) + return accelerators, tuple(lane.name for lane in counted) + + total, names = held(plan.lanes) + uses = [] + if capacity.total is not None: + uses.append(CapacityUse(scope=CapacityScope.TOTAL, ceiling=capacity.total, accelerators=total, lanes=names)) + for ceiling in sorted(capacity.types, key=lambda entry: entry.gpu_product): + accelerators, lanes = held( + lane for lane in plan.lanes if any(cell.gpu_product == ceiling.gpu_product for cell in lane.cells) + ) + uses.append( + CapacityUse( + scope=CapacityScope.TYPE, + gpu_product=ceiling.gpu_product, + ceiling=ceiling.accelerators, + accelerators=accelerators, + lanes=lanes, + ) + ) + if capacity.shared_quota is not None: + quota = capacity.shared_quota + uses.append( + CapacityUse( + scope=CapacityScope.SHARED_QUOTA, + ceiling=quota.accelerators, + accelerators=total + quota.held_elsewhere, + held_elsewhere=quota.held_elsewhere, + lanes=names, + ) + ) + return tuple(uses) + @staticmethod def lane_names(placed: Mapping[str, _Option], reserved: set[str]) -> dict[str, str]: bases = { - identity: _slug(f"{identity}-{option.receipt.cell.machine_profile}") or "lane" + identity: _slug("-".join((identity, *(cell.machine_profile for cell in option.cells)))) or "lane" for identity, option in placed.items() } counts = Counter(bases.values()) @@ -882,6 +1199,7 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: queue_pool=self.parameters.queue_pool, bundle=self.parameters.bundle_of(identity), cell=receipt.cell, + fallback_cells=option.cells[1:], container_shape=receipt.container_shape, concurrency=receipt.concurrency, max_containers=self.parameters.max_containers, @@ -917,6 +1235,7 @@ def kept_blockers( blockers.extend(() if blocker is None else (blocker,)) elif boot is not None or level.come_alive_p95_s == 0: blockers.append(Blocker(field="comeAliveP95S", bound=level.come_alive_p95_s, measured=None)) + blockers.extend(self.reliability_blockers(level, lane.cells)) return tuple(blockers) def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) -> float | None: @@ -973,15 +1292,17 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: lanes=tuple(lanes[name] for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) - receipts: dict[str, Receipt | None] = {identity: option.receipt for identity, option in placed.items()} + receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) lane_reports, costs = self._pricing(new_plan, receipts) total = sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) reports: list[IdentityReport] = [] requests: list[MeasurementRequest] = [] levers: list[Lever] = [] + scarcity: list[Scarcity] = [] for choice in choices: baseline = choice.baseline + scarcity.extend(self.scarcity(choice)) requests.extend(choice.gaps) requests.extend(self.receipt_requests(choice, baseline)) requests.extend(self.boot_requests(choice, baseline)) @@ -1015,6 +1336,8 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: for identity in sorted(self.levels) if not _declares_warm_bound(self.levels[identity]) and self.receipts.get(identity) ), + capacity=self.capacity_use(new_plan), + scarcity=tuple(sorted(scarcity, key=_scarcity_rank)), ) return PlannerResult(plan=new_plan, report=report) @@ -1050,7 +1373,7 @@ def _identity_report( next_cheaper=next_cheaper, ) return replace(report, unplaced_reason=reason) - rate = chosen.receipt.throughput.requests_per_second + rate = chosen.pricing_receipt.throughput.requests_per_second return replace( report, status=IdentityStatus.PLACED, @@ -1072,13 +1395,11 @@ def _pricing( costs: dict[str, float] = {} for lane in plan.lanes: hosted = tuple(sorted(residents[lane.name])) - reason = _refusal_reason(lane.cell, self.prices) + reason = _lane_refusal(lane, self.prices) if reason is not None: reports.append(LaneReport(name=lane.name, identities=hosted, unpriced_reason=reason)) continue - hourly = container_hourly_cost( - self.prices[lane.cell.machine_profile], lane.container_shape, self.multiplier - ) + hourly = self.lane_hourly(lane) floor = lane.envelope.floor_min * hours demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} @@ -1097,6 +1418,17 @@ def _pricing( return tuple(reports), costs +def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | None: + fields = _fields(option.blockers) + if fields == [PLACEMENT_FAILURE_FIELD]: + return ScarcityReason.UNRELIABLE_TYPE + if option.warmth not in COLD_WARMTHS or fields != ["comeAliveP95S"] or not option.placement_wait_s: + return None + if not bound or option.boot_s is None or option.boot_s > bound: + return None + return ScarcityReason.PLACEMENT_WAIT + + def _saving(total: float, replanned: float) -> float: saving = total - replanned return saving if round(saving, 2) > 0 else 0.0 @@ -1165,6 +1497,18 @@ def _lever_rank(lever: Lever) -> tuple[object, ...]: ) +def _scarcity_rank(entry: Scarcity) -> tuple[object, ...]: + return ( + entry.identity, + entry.reason.value, + entry.cell.machine_profile, + entry.cell.gpu_product, + _shape_key(entry.container_shape), + entry.warmth.value, + entry.lane or "", + ) + + def _refusal_rank(refusal: Refusal) -> tuple[str, ...]: return ( refusal.record.value, @@ -1185,5 +1529,12 @@ def plan( demand: Demand, current_plan: Plan | None, parameters: PlannerParameters, + *, + capacity: Capacity | None = None, ) -> PlannerResult: - return _Planner(service_levels, receipts, boot_evidence, price_list, demand, current_plan, parameters).run() + result = _Planner( + service_levels, receipts, boot_evidence, price_list, demand, current_plan, parameters, capacity + ).run() + if any(use.exceeded for use in result.report.capacity): + raise CapacityExceededError(result) + return result diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 585f5f8d0..d63ffc25d 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -33,6 +33,7 @@ class UnplacedReason(StrEnum): WARM_BOUND_UNDECLARED = "warm-bound-undeclared" CURRENT_UNMEASURED = "current-unmeasured" CURRENT_UNPRICED = "current-unpriced" + PLACEMENT_UNRELIABLE = "placement-unreliable" class ComeAliveBasis(StrEnum): @@ -50,6 +51,7 @@ class RequestReason(StrEnum): UNMEASURED = "unmeasured" REFUSED = "refused" FAILED = "failed" + UNPLACED = "unplaced" class RefusedRecord(StrEnum): @@ -57,6 +59,18 @@ class RefusedRecord(StrEnum): BOOT_EVIDENCE = "boot-evidence" +class ScarcityReason(StrEnum): + PLACEMENT_WAIT = "placement-wait" + UNPLACED = "unplaced" + UNRELIABLE_TYPE = "unreliable-type" + + +class CapacityScope(StrEnum): + TOTAL = "total" + TYPE = "type" + SHARED_QUOTA = "shared-quota" + + def _money(value: float) -> float: return round(value, 2) + 0.0 @@ -72,7 +86,7 @@ def _compact(fields: dict[str, object | None]) -> dict[str, object]: @dataclass(frozen=True, slots=True, kw_only=True) class Blocker: field: str - bound: int | None + bound: float | None measured: float | None def to_dict(self) -> dict[str, object]: @@ -92,11 +106,14 @@ class Option: blocked_by: tuple[Blocker, ...] = () saving: float | None = None lane: str | None = None + fallback_cells: tuple[Cell, ...] = () + placement_wait_s: float | None = None def to_dict(self) -> dict[str, object]: return _compact( { "cell": self.cell.to_dict(), + "fallbackCells": [cell.to_dict() for cell in self.fallback_cells] or None, "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, @@ -104,6 +121,7 @@ def to_dict(self) -> dict[str, object]: "monthlyCost": _money(self.monthly_cost), "saving": None if self.saving is None else _money(self.saving), "comeAliveS": self.come_alive_s, + "placementWaitS": self.placement_wait_s, "comeAliveBasis": self.come_alive_basis.value, "blockedBy": [blocker.to_dict() for blocker in self.blocked_by] or None, "receipt": self.receipt.to_dict(), @@ -236,21 +254,102 @@ class Lever: saving_under_bound: float permitted_bound_s: int saving_at_permitted_bound: float + placement_wait_s: float | None = None def to_dict(self) -> dict[str, object]: - return { - "identity": self.identity, - "cell": self.cell.to_dict(), - "containerShape": self.container_shape.to_dict(), - "warmth": self.warmth.value, - "comeAliveS": self.come_alive_s, - "comeAliveBasis": self.come_alive_basis.value, - "comeAliveBoundS": self.bound_s, - "monthlyCost": _money(self.monthly_cost), - "savingUnderBound": _money(self.saving_under_bound), - "permittedBoundS": self.permitted_bound_s, - "savingAtPermittedBound": _money(self.saving_at_permitted_bound), - } + return _compact( + { + "identity": self.identity, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "warmth": self.warmth.value, + "comeAliveS": self.come_alive_s, + "placementWaitS": self.placement_wait_s, + "comeAliveBasis": self.come_alive_basis.value, + "comeAliveBoundS": self.bound_s, + "monthlyCost": _money(self.monthly_cost), + "savingUnderBound": _money(self.saving_under_bound), + "permittedBoundS": self.permitted_bound_s, + "savingAtPermittedBound": _money(self.saving_at_permitted_bound), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Scarcity: + identity: str + reason: ScarcityReason + cell: Cell + container_shape: ContainerShape + warmth: Warmth + monthly_cost: float + placement_failure_rate: float + come_alive_bound_s: int | None = None + boot_s: float | None = None + placement_wait_s: float | None = None + lane: str | None = None + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "identity": self.identity, + "reason": self.reason.value, + "cell": self.cell.to_dict(), + "containerShape": self.container_shape.to_dict(), + "lane": self.lane, + "warmth": self.warmth.value, + "monthlyCost": _money(self.monthly_cost), + "comeAliveBoundS": self.come_alive_bound_s, + "bootS": self.boot_s, + "placementWaitS": self.placement_wait_s, + "placementFailureRate": _rate(self.placement_failure_rate), + } + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CapacityUse: + scope: CapacityScope + ceiling: int + accelerators: int + lanes: tuple[str, ...] + gpu_product: str | None = None + held_elsewhere: int | None = None + + @property + def exceeded(self) -> bool: + return self.accelerators > self.ceiling + + def describe(self) -> str: + lanes = ", ".join(self.lanes) or "no lane" + if self.scope is CapacityScope.TYPE: + return ( + f"the envelope maxima of {lanes} hold {self.accelerators} {self.gpu_product} accelerators, " + f"above the type ceiling of {self.ceiling}" + ) + if self.scope is CapacityScope.SHARED_QUOTA: + elsewhere = self.held_elsewhere or 0 + return ( + f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " + f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" + ) + return ( + f"the envelope maxima of {lanes} hold {self.accelerators} accelerators, " + f"above the total ceiling of {self.ceiling}" + ) + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "scope": self.scope.value, + "gpuProduct": self.gpu_product, + "ceiling": self.ceiling, + "accelerators": self.accelerators, + "heldElsewhere": self.held_elsewhere, + "lanes": list(self.lanes), + "exceeded": self.exceeded, + } + ) @dataclass(frozen=True, slots=True, kw_only=True) @@ -318,6 +417,8 @@ class Report: identity_changes: tuple[IdentityChange, ...] lane_changes: tuple[LaneChange, ...] warm_bound_undeclared: tuple[str, ...] = () + capacity: tuple[CapacityUse, ...] = () + scarcity: tuple[Scarcity, ...] = () @property def monthly_cost(self) -> float: @@ -355,9 +456,11 @@ def to_dict(self) -> dict[str, object]: }, "identities": [entry.to_dict() for entry in self.identities], "lanes": [lane.to_dict() for lane in self.lanes], + "capacity": [use.to_dict() for use in self.capacity], "sharedChartLanes": [shared.to_dict() for shared in self.shared_chart_lanes], "measurementRequests": [request.to_dict() for request in self.measurement_requests], "comeAliveLevers": [lever.to_dict() for lever in self.come_alive_levers], + "scarcity": [entry.to_dict() for entry in self.scarcity], "queueWaitUnattained": list(self.queue_wait_unattained), "warmBoundUndeclared": list(self.warm_bound_undeclared), "refusals": [refusal.to_dict() for refusal in self.refusals], diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index ba9e691ba..93b9d9cd0 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -17,6 +17,7 @@ BOOT_EVIDENCE_KIND = "placement-boot-evidence" PRICE_LIST_KIND = "placement-price-list" DEMAND_KIND = "placement-demand" +CAPACITY_KIND = "placement-capacity" PLAN_KIND = "placement-plan" ROOT_PATH = "$" @@ -57,6 +58,7 @@ class BootOutcome(StrEnum): SUCCEEDED = "succeeded" FAILED = "failed" TIMED_OUT = "timed-out" + UNPLACED = "unplaced" def _join(path: str, key: str) -> str: @@ -667,9 +669,10 @@ class BootEvidence: cell: Cell container_shape: ContainerShape warmth: Warmth - boot_seconds: float outcome: BootOutcome source: SourceRef + boot_seconds: float | None = None + placement_wait_s: float | None = None borrowed: bool = False def __post_init__(self) -> None: @@ -677,6 +680,17 @@ def __post_init__(self) -> None: if self.warmth not in BOOT_WARMTHS: choices = ", ".join(repr(warmth.value) for warmth in Warmth if warmth in BOOT_WARMTHS) raise SchemaError("warmth", f"boot evidence records one of {choices}, got {self.warmth.value!r}") + if self.warmth is Warmth.LAZY_RESIDENT: + if self.outcome is BootOutcome.UNPLACED: + raise SchemaError("outcome", "a lazy-resident load runs on a placed container and is never unplaced") + _require_absent( + "placementWaitS", self.placement_wait_s, "for a lazy-resident load, which runs on a placed container" + ) + if self.outcome is BootOutcome.UNPLACED: + _require_absent("bootSeconds", self.boot_seconds, "for an unplaced attempt, which never booted") + elif self.boot_seconds is None: + raise SchemaError("bootSeconds", "is required for an attempt that was placed") + _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) @classmethod @@ -689,7 +703,8 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: cell=reader.child("cell", Cell.from_dict), container_shape=reader.child("containerShape", ContainerShape.from_dict), warmth=reader.enum("warmth", Warmth), - boot_seconds=reader.number("bootSeconds"), + placement_wait_s=reader.optional_number("placementWaitS"), + boot_seconds=reader.optional_number("bootSeconds"), outcome=reader.enum("outcome", BootOutcome), borrowed=reader.optional_boolean("borrowed") is True, source=reader.child("source", SourceRef.from_dict), @@ -704,6 +719,7 @@ def to_dict(self) -> dict[str, object]: "cell": self.cell.to_dict(), "containerShape": self.container_shape.to_dict(), "warmth": self.warmth.value, + "placementWaitS": self.placement_wait_s, "bootSeconds": self.boot_seconds, "outcome": self.outcome.value, "borrowed": True if self.borrowed else None, @@ -900,6 +916,81 @@ def to_dict(self) -> dict[str, object]: } +@dataclass(frozen=True, slots=True, kw_only=True) +class TypeCeiling: + gpu_product: str + accelerators: int + + def __post_init__(self) -> None: + _require_identifier("gpuProduct", self.gpu_product) + _require_at_least("accelerators", self.accelerators, 0) + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + return reader.build(cls, gpu_product=reader.string("gpuProduct"), accelerators=reader.integer("accelerators")) + + def to_dict(self) -> dict[str, object]: + return {"gpuProduct": self.gpu_product, "accelerators": self.accelerators} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SharedQuota: + accelerators: int + held_elsewhere: int = 0 + + def __post_init__(self) -> None: + _require_at_least("accelerators", self.accelerators, 0) + _require_at_least("heldElsewhere", self.held_elsewhere, 0) + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + return reader.build( + cls, + accelerators=reader.integer("accelerators"), + held_elsewhere=reader.optional_integer("heldElsewhere") or 0, + ) + + def to_dict(self) -> dict[str, object]: + return {"accelerators": self.accelerators, "heldElsewhere": self.held_elsewhere} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Capacity: + total: int | None = None + types: tuple[TypeCeiling, ...] = () + shared_quota: SharedQuota | None = None + + def __post_init__(self) -> None: + _require_at_least("total", self.total, 0) + _require_unique("types", "gpuProduct", (ceiling.gpu_product for ceiling in self.types)) + if self.total is None and not self.types and self.shared_quota is None: + raise SchemaError("", "must declare total, types or sharedQuota") + + @classmethod + def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: + reader = _Reader(data, path) + reader.header(CAPACITY_KIND) + return reader.build( + cls, + total=reader.optional_integer("total"), + types=reader.children("types", TypeCeiling.from_dict, required=False), + shared_quota=reader.optional_child("sharedQuota", SharedQuota.from_dict), + ) + + def to_dict(self) -> dict[str, object]: + return _compact( + { + "kind": CAPACITY_KIND, + "schemaVersion": SCHEMA_VERSION, + "total": self.total, + "types": [ceiling.to_dict() for ceiling in self.types] or None, + "sharedQuota": None if self.shared_quota is None else self.shared_quota.to_dict(), + } + ) + + @dataclass(frozen=True, slots=True, kw_only=True) class Envelope: floor_min: int @@ -959,6 +1050,7 @@ class PlanLane: concurrency: int max_containers: int envelope: Envelope + fallback_cells: tuple[Cell, ...] = () def __post_init__(self) -> None: _require_identifier("name", self.name) @@ -966,12 +1058,22 @@ def __post_init__(self) -> None: _require_identifier("bundle", self.bundle) _require_at_least("concurrency", self.concurrency, 1) _require_at_least("maxContainers", self.max_containers, 1) + for index, cell in enumerate(self.fallback_cells): + if cell in self.cells[: index + 1]: + raise SchemaError( + f"fallbackCells[{index}]", + f"repeats {cell.machine_profile!r} with {cell.gpu_product!r}, already a cell of this lane", + ) if self.envelope.floor_max > self.max_containers: raise SchemaError( "envelope.floorMax", f"must not exceed maxContainers ({self.max_containers}), got {self.envelope.floor_max}", ) + @property + def cells(self) -> tuple[Cell, ...]: + return (self.cell, *self.fallback_cells) + @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: reader = _Reader(data, path) @@ -981,6 +1083,7 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: queue_pool=reader.string("queuePool"), bundle=reader.string("bundle"), cell=reader.child("cell", Cell.from_dict), + fallback_cells=reader.children("fallbackCells", Cell.from_dict, required=False), container_shape=reader.child("containerShape", ContainerShape.from_dict), concurrency=reader.integer("concurrency"), max_containers=reader.integer("maxContainers"), @@ -988,16 +1091,23 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: ) def to_dict(self) -> dict[str, object]: - return { + document: dict[str, object] = { "name": self.name, "queuePool": self.queue_pool, "bundle": self.bundle, "cell": self.cell.to_dict(), - "containerShape": self.container_shape.to_dict(), - "concurrency": self.concurrency, - "maxContainers": self.max_containers, - "envelope": self.envelope.to_dict(), } + if self.fallback_cells: + document["fallbackCells"] = [cell.to_dict() for cell in self.fallback_cells] + document.update( + { + "containerShape": self.container_shape.to_dict(), + "concurrency": self.concurrency, + "maxContainers": self.max_containers, + "envelope": self.envelope.to_dict(), + } + ) + return document @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/sie_placement/tests/fixtures/boot-evidence.json b/packages/sie_placement/tests/fixtures/boot-evidence.json index be9e58814..73648db0a 100644 --- a/packages/sie_placement/tests/fixtures/boot-evidence.json +++ b/packages/sie_placement/tests/fixtures/boot-evidence.json @@ -12,6 +12,7 @@ "memoryMiB": 87040 }, "warmth": "cold-with-bound", + "placementWaitS": 41.2, "bootSeconds": 187.4, "outcome": "succeeded", "source": { diff --git a/packages/sie_placement/tests/fixtures/capacity.json b/packages/sie_placement/tests/fixtures/capacity.json new file mode 100644 index 000000000..da79b0646 --- /dev/null +++ b/packages/sie_placement/tests/fixtures/capacity.json @@ -0,0 +1,19 @@ +{ + "kind": "placement-capacity", + "schemaVersion": 1, + "total": 12, + "types": [ + { + "gpuProduct": "NVIDIA-L4", + "accelerators": 8 + }, + { + "gpuProduct": "NVIDIA-A100-SXM4-80GB", + "accelerators": 4 + } + ], + "sharedQuota": { + "accelerators": 16, + "heldElsewhere": 6 + } +} diff --git a/packages/sie_placement/tests/fixtures/plan.json b/packages/sie_placement/tests/fixtures/plan.json index ed9a2f5fa..940112335 100644 --- a/packages/sie_placement/tests/fixtures/plan.json +++ b/packages/sie_placement/tests/fixtures/plan.json @@ -35,6 +35,12 @@ "machineProfile": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB" }, + "fallbackCells": [ + { + "machineProfile": "h100", + "gpuProduct": "NVIDIA-H100-80GB-HBM3" + } + ], "containerShape": { "gpuCount": 1, "cpuCores": 12, diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 076b55a12..de243df5b 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -9,6 +9,7 @@ "fallbackFloor": 2, "cooldownS": 900, "bufferContainers": 1, + "holdBudgetS": 0, "hoursPerMonth": 730.0, "queuePool": "default", "bundle": "default", @@ -323,6 +324,7 @@ "envelopeMaxMonthlyCost": 1518.4 } ], + "capacity": [], "sharedChartLanes": [ { "queuePool": "default", @@ -525,6 +527,7 @@ "savingAtPermittedBound": 379.6 } ], + "scarcity": [], "queueWaitUnattained": [ "BAAI/bge-m3", "BAAI/bge-m3:sparse", diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py new file mode 100644 index 000000000..eac0a3852 --- /dev/null +++ b/packages/sie_placement/tests/test_availability.py @@ -0,0 +1,781 @@ +from __future__ import annotations + +import math +import random +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import pytest +from sie_placement.planner import CapacityExceededError, PlannerInputError, PlannerParameters, plan +from sie_placement.report import ( + Blocker, + CapacityScope, + ComeAliveBasis, + IdentityStatus, + Measurement, + RequestReason, + ScarcityReason, + UnplacedReason, + render, +) +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Capacity, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + RequestShape, + ServiceLevel, + ServiceLevels, + SharedQuota, + SourceRef, + Throughput, + TypeCeiling, + Warmth, +) + +START = datetime(2026, 11, 2, tzinfo=UTC) +MONTH = 730.0 +SCALE = MONTH / 24 +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +H100 = Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3") +RATES = {L4: 1.0, A100: 3.0, H100: 4.0} +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) +IDENTITY = "org/model" +OTHER = "org/other" +PRICES = PriceList( + currency="USD", + machine_profiles=tuple( + MachineProfilePrice( + name=cell.machine_profile, + gpu_product=cell.gpu_product, + accelerator_per_hour=rate, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + ) + for cell, rate in RATES.items() + ), + placements=(Placement(name="test", region_multiplier=1),), +) +UNRELIABLE = 1 / 3 + + +def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=rps), + latency=Percentiles(p99_ms=p99), + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{rps}/{p99}"), + ) + + +def attempt( + identity=IDENTITY, + cell=L4, + *, + warmth=Warmth.COLD_WITH_BOUND, + boot=60.0, + wait=None, + outcome=BootOutcome.SUCCEEDED, + borrowed=False, + shape=SHAPE, + index=0, +) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=shape, + warmth=warmth, + outcome=outcome, + boot_seconds=None if outcome is BootOutcome.UNPLACED else boot, + placement_wait_s=wait, + borrowed=borrowed, + source=SourceRef(uri=f"boots/{identity}/{cell.machine_profile}/{warmth.value}/{index}"), + ) + + +def unplaced(cell=L4, *, index=0, identity=OTHER) -> BootEvidence: + return attempt(identity, cell, outcome=BootOutcome.UNPLACED, index=index) + + +def scarce(cell=L4, *, placed=2, failed=1) -> list[BootEvidence]: + rows = [attempt(OTHER, cell, boot=30.0, index=index) for index in range(placed)] + rows.extend(unplaced(cell, index=placed + index) for index in range(failed)) + return rows + + +def levels(**bounds) -> ServiceLevels: + return ServiceLevels( + identities=tuple( + IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + for identity, bound in sorted(bounds.items()) + ) + ) + + +def one(bound) -> ServiceLevels: + return levels(**{IDENTITY: bound}) + + +def demand(hourly=None) -> Demand: + return Demand( + start=START, + end=START + timedelta(hours=24), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=count) + for hour, count in sorted(counts.items()) + ), + ) + for identity, counts in sorted((hourly or {}).items()) + ), + ) + + +def parameters(**overrides) -> PlannerParameters: + values = {"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides + return PlannerParameters(**values) + + +def run(service_levels, receipts=(), boots=(), *, hourly=None, current=None, capacity=None, **overrides): + return plan( + service_levels, receipts, boots, PRICES, demand(hourly), current, parameters(**overrides), capacity=capacity + ) + + +def rendered(result) -> tuple[str, str]: + return render(result.plan.to_dict()), render(result.report.to_dict()) + + +def lane(name, cell=L4, *, fallbacks=(), floor=1, buffer=0) -> PlanLane: + return PlanLane( + name=name, + queue_pool="default", + bundle="default", + cell=cell, + fallback_cells=tuple(fallbacks), + container_shape=SHAPE, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=floor, floor_max=floor + buffer), + ) + + +def plan_of(lanes, residents) -> Plan: + return Plan( + placement="test", + lanes=tuple(lanes), + identities=tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) + for identity, (name, warmth) in sorted(residents.items()) + ), + ) + + +def test_a_cold_come_alive_adds_the_placement_wait_to_the_boot(): + result = run(one(120), [receipt()], [attempt(boot=50.0, wait=30.0)]) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.come_alive_s, chosen.placement_wait_s) == (Warmth.COLD_WITH_BOUND, 80.0, 30.0) + assert chosen.to_dict()["placementWaitS"] == 30.0 + assert result.report.scarcity == () + + +def test_placement_waits_pool_over_every_attempt_on_a_gpu_type(): + boots = [ + attempt(boot=50.0), + attempt(OTHER, boot=500.0, wait=100.0, index=1), + attempt(OTHER, A100, boot=10.0, wait=900.0, index=2), + ] + result = run(one(120), [receipt()], boots) + report = result.report.identity(IDENTITY) + assert report.chosen.warmth is Warmth.PINNED + assert report.next_cheaper.blocked_by == (Blocker(field="comeAliveP95S", bound=120, measured=150.0),) + assert report.next_cheaper.placement_wait_s == 100.0 + (entry,) = result.report.scarcity + assert (entry.identity, entry.reason, entry.cell, entry.warmth) == ( + IDENTITY, + ScarcityReason.PLACEMENT_WAIT, + L4, + Warmth.COLD_WITH_BOUND, + ) + assert (entry.boot_s, entry.placement_wait_s, entry.come_alive_bound_s, entry.monthly_cost) == (50.0, 100.0, 120, 0) + assert entry.to_dict()["reason"] == "placement-wait" + + +@pytest.mark.parametrize(("fast", "come_alive"), [(18, None), (19, 60.0), (37, None), (38, 60.0)]) +def test_the_placement_wait_takes_the_95th_percentile_from_twenty_attempts(fast, come_alive): + boots = [attempt(boot=50.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(fast)) + boots.extend(attempt(OTHER, boot=1.0, wait=200.0, index=100 + index) for index in range(1 if fast < 30 else 2)) + chosen = run(one(120), [receipt()], boots).report.identity(IDENTITY).chosen + if come_alive is None: + assert chosen.warmth is Warmth.PINNED + else: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, come_alive) + + +@pytest.mark.parametrize(("placed", "selected"), [(17, False), (18, True)]) +def test_an_unplaced_attempt_ranks_above_every_placed_one(placed, selected): + boots = [attempt(boot=50.0, wait=10.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(placed)) + boots.append(unplaced(index=99)) + result = run(one(120), [receipt()], boots) + chosen = result.report.identity(IDENTITY).chosen + if selected: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, 60.0) + assert result.report.scarcity == () + return + assert chosen.warmth is Warmth.PINNED + (request,) = [request for request in result.report.measurement_requests if request.cell == L4] + assert (request.measure, request.warmth, request.reason) == ( + (Measurement.BOOT_EVIDENCE,), + Warmth.COLD_WITH_BOUND, + RequestReason.UNPLACED, + ) + assert request.to_dict()["reason"] == "unplaced" + (entry,) = result.report.scarcity + assert (entry.reason, entry.boot_s, entry.placement_wait_s) == (ScarcityReason.UNPLACED, 50.0, None) + assert entry.placement_failure_rate == pytest.approx(1 / (placed + 2)) + + +def test_an_attempt_without_a_placement_wait_leaves_its_boot_to_stand_for_the_come_alive(): + result = run(one(120), [receipt()], [attempt(boot=95.0)]) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.placement_wait_s) == (95.0, None) + assert "placementWaitS" not in chosen.to_dict() + + +def test_own_attempts_on_a_type_replace_borrowed_ones(): + boots = [attempt(OTHER, boot=1.0, wait=200.0, borrowed=True, index=1), attempt(boot=50.0)] + chosen = run(one(120), [receipt()], boots).report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.come_alive_basis) == (50.0, ComeAliveBasis.BOOT_EVIDENCE) + borrowed = [attempt(boot=50.0, wait=20.0, borrowed=True)] + chosen = run(one(120), [receipt()], borrowed).report.identity(IDENTITY).chosen + assert (chosen.come_alive_s, chosen.come_alive_basis) == (70.0, ComeAliveBasis.BORROWED_BOOT_EVIDENCE) + + +def test_an_unplaced_attempt_of_the_identity_withdraws_its_cold_cell_until_it_boots(): + result = run(one(120), [receipt()], [unplaced(identity=IDENTITY)]) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + (request,) = [request for request in result.report.measurement_requests if request.cell == L4] + assert request.reason is RequestReason.UNPLACED + assert result.report.scarcity == () + + +def test_a_snapshot_restore_waits_for_placement_too(): + boots = [attempt(warmth=Warmth.SNAPSHOT_COLD, boot=5.0, wait=30.0)] + result = run(one(20), [receipt()], boots) + report = result.report.identity(IDENTITY) + assert report.chosen.warmth is Warmth.PINNED + assert (report.next_cheaper.warmth, report.next_cheaper.come_alive_s) == (Warmth.SNAPSHOT_COLD, 35.0) + assert [(entry.warmth, entry.reason) for entry in result.report.scarcity] == [ + (Warmth.SNAPSHOT_COLD, ScarcityReason.PLACEMENT_WAIT) + ] + + +def test_a_lazy_resident_load_waits_for_no_placement(): + current = plan_of( + [lane("own"), lane("shared")], {OTHER: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} + ) + service_levels = levels(**{OTHER: 0, IDENTITY: 60}) + boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), attempt(OTHER, boot=1.0, wait=500.0, index=1)] + result = run(service_levels, [receipt(OTHER), receipt()], boots, current=current) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.warmth, chosen.lane, chosen.come_alive_s, chosen.placement_wait_s) == ( + Warmth.LAZY_RESIDENT, + "shared", + 20.0, + None, + ) + + +@pytest.mark.parametrize(("wait", "under"), [(30.0, MONTH), (70.0, 0.0)]) +def test_a_lever_under_its_bound_leaves_room_for_the_placement_wait(wait, under): + result = run(one(60), [receipt()], [attempt(boot=50.0, wait=wait)]) + (lever,) = result.report.come_alive_levers + assert (lever.come_alive_s, lever.placement_wait_s, lever.permitted_bound_s) == (50.0 + wait, wait, 50 + wait) + assert lever.saving_at_permitted_bound == pytest.approx(MONTH) + assert lever.saving_under_bound == pytest.approx(under) + assert lever.to_dict()["placementWaitS"] == wait + + +def test_an_unreliable_type_is_not_chosen_under_a_bound_the_hold_budget_fits(): + result = run(one(0), [receipt(), receipt(cell=A100)], scarce()) + report = result.report.identity(IDENTITY) + assert (report.chosen.cell, report.chosen.warmth) == (A100, Warmth.PINNED) + assert report.chosen.monthly_cost == pytest.approx(3 * MONTH) + assert report.next_cheaper.cell == L4 + assert report.next_cheaper.blocked_by == ( + Blocker(field="placementFailureRate", bound=0.05, measured=pytest.approx(UNRELIABLE)), + ) + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + (entry,) = result.report.scarcity + assert (entry.reason, entry.cell, entry.warmth, entry.monthly_cost) == ( + ScarcityReason.UNRELIABLE_TYPE, + L4, + Warmth.PINNED, + pytest.approx(MONTH), + ) + assert entry.to_dict()["placementFailureRate"] == 0.3333 + assert result.report.to_dict()["identities"][0]["nextCheaper"]["blockedBy"][0]["field"] == "placementFailureRate" + + +def test_loosening_a_bound_above_the_hold_budget_admits_an_unreliable_type(): + report = run(one(0), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=20).report.identity(IDENTITY) + loosened = run(one(21), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=20).report + assert report.chosen.cell == A100 + assert loosened.identity(IDENTITY).chosen.cell == L4 + assert report.next_cheaper.saving == pytest.approx(2 * MONTH) + + +@pytest.mark.parametrize(("hold", "cell"), [(20, A100), (19, L4), (0, L4)]) +def test_the_hold_budget_decides_which_bounds_avoid_unreliable_types(hold, cell): + result = run(one(20), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=hold) + assert result.report.identity(IDENTITY).chosen.cell == cell + assert bool(result.report.scarcity) is (cell == A100) + + +def test_an_undeclared_come_alive_bound_may_use_an_unreliable_type(): + result = run(one(None), [receipt(), receipt(cell=A100)], scarce(), hold_budget_s=600) + assert result.report.identity(IDENTITY).chosen.cell == L4 + + +def test_a_placement_failure_rate_of_five_percent_is_reliable(): + boots = scarce(placed=19, failed=1) + assert run(one(0), [receipt(), receipt(cell=A100)], boots).report.identity(IDENTITY).chosen.cell == L4 + boots = scarce(placed=18, failed=1) + assert run(one(0), [receipt(), receipt(cell=A100)], boots).report.identity(IDENTITY).chosen.cell == A100 + + +def test_the_failure_rate_counts_the_environments_own_attempts_over_borrowed_ones(): + borrowed = [replace(row, borrowed=True) for row in scarce(placed=0, failed=3)] + own = [attempt(OTHER, boot=30.0, index=10)] + result = run(one(0), [receipt(), receipt(cell=A100)], [*borrowed, *own]) + assert result.report.identity(IDENTITY).chosen.cell == L4 + result = run(one(0), [receipt(), receipt(cell=A100)], borrowed) + assert result.report.identity(IDENTITY).chosen.cell == A100 + + +def test_an_identity_whose_only_types_are_unreliable_is_unplaced_and_says_why(): + result = run(one(0), [receipt()], scarce()) + report = result.report.identity(IDENTITY) + assert (report.status, report.unplaced_reason) == (IdentityStatus.UNPLACED, UnplacedReason.PLACEMENT_UNRELIABLE) + assert report.to_dict()["unplacedReason"] == "placement-unreliable" + assert result.plan.identities == () + + +def test_unreliable_types_are_not_requested_for_a_bound_the_hold_budget_fits(): + result = run(one(0), [receipt(cell=H100)], scarce()) + assert [request.cell for request in result.report.measurement_requests] == [A100] + result = run(one(600), [receipt(cell=H100)], scarce()) + assert sorted(request.cell.machine_profile for request in result.report.measurement_requests) == [ + "a100-80gb", + "h100", + "l4", + ] + + +def test_a_current_placement_on_an_unreliable_type_is_known_to_miss_its_bound(): + current = plan_of([lane("own")], {IDENTITY: ("own", Warmth.PINNED)}) + moved = run(one(0), [receipt(), receipt(cell=A100)], scarce(), current=current) + assert moved.report.identity(IDENTITY).chosen.cell == A100 + assert [entry.lane for entry in moved.plan.identities] == ["org-model-a100-80gb"] + kept = run(one(0), [receipt()], scarce(), current=current) + report = kept.report.identity(IDENTITY) + assert (report.status, report.lane, report.unplaced_reason) == ( + IdentityStatus.KEPT, + "own", + UnplacedReason.PLACEMENT_UNRELIABLE, + ) + assert report.blocked_by == (Blocker(field="placementFailureRate", bound=0.05, measured=pytest.approx(UNRELIABLE)),) + assert kept.plan == current + + +def test_a_candidate_list_names_a_second_type_when_every_single_type_is_unreliable(): + boots = [*scarce(L4), *scarce(A100)] + receipts = [receipt(), receipt(cell=A100)] + result = run(one(0), receipts, boots) + chosen = result.report.identity(IDENTITY).chosen + assert (chosen.cell, chosen.fallback_cells, chosen.warmth) == (L4, (A100,), Warmth.PINNED) + assert chosen.monthly_cost == pytest.approx(3 * MONTH) + assert chosen.to_dict()["fallbackCells"] == [A100.to_dict()] + (planned,) = result.plan.lanes + assert (planned.name, planned.cell, planned.fallback_cells) == ("org-model-l4-a100-80gb", L4, (A100,)) + assert result.report.monthly_cost == pytest.approx(3 * MONTH) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + again = run(one(0), receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_candidate_list_is_not_offered_where_a_reliable_type_is_cheaper_or_as_cheap(): + result = run(one(0), [receipt(), receipt(cell=A100)], scarce(L4)) + assert result.report.identity(IDENTITY).chosen.fallback_cells == () + assert result.plan.lanes[0].fallback_cells == () + + +@pytest.mark.parametrize( + "second", + [ + receipt(cell=A100, p99=300.0), + receipt(cell=A100, concurrency=16), + receipt(cell=A100, shape=PAIR), + ], +) +def test_a_list_member_meets_the_warm_bound_at_the_lane_shape_and_concurrency(second): + result = run(one(0), [receipt(), second], [*scarce(L4), *scarce(A100)]) + assert result.report.identity(IDENTITY).status is IdentityStatus.UNPLACED + + +def test_a_candidate_list_is_priced_at_its_dearest_member_and_its_slowest_receipt(): + receipts = [receipt(rps=10.0), receipt(cell=A100, rps=5.0)] + result = run(one(0), receipts, [*scarce(L4), *scarce(A100)], hourly={IDENTITY: {0: 100000}}) + report = result.report.identity(IDENTITY) + above = 3 * SCALE + assert report.chosen.monthly_cost == pytest.approx(3 * (MONTH + above)) + assert report.monthly_cost == pytest.approx(3 * (MONTH + above)) + assert report.peak_containers == 6 + (lane_report,) = result.report.lanes + assert lane_report.hourly_cost == 3.0 + + +def test_a_current_candidate_list_stays_when_a_plain_type_costs_the_same(): + current = plan_of([lane("listed", fallbacks=(A100,))], {IDENTITY: ("listed", Warmth.PINNED)}) + result = run(one(0), [receipt(), receipt(cell=A100)], scarce(L4), current=current) + assert result.plan == current + assert result.report.unchanged + assert result.report.identity(IDENTITY).chosen.fallback_cells == (A100,) + + +def test_a_lane_with_fallback_cells_hosts_no_lazy_resident(): + current = plan_of( + [lane("own"), lane("listed", fallbacks=(A100,))], + {OTHER: ("listed", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + service_levels = levels(**{OTHER: 0, IDENTITY: 60}) + boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), *scarce(L4)] + result = run(service_levels, [receipt(OTHER), receipt(OTHER, A100), receipt()], boots, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities}[OTHER] == "listed" + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + + +def two_lanes(**overrides): + receipts = [receipt("org/a"), receipt("org/b", A100)] + return run(levels(**{"org/a": 0, "org/b": 0}), receipts, buffer_containers=1, **overrides) + + +def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): + capacity = Capacity( + total=4, + types=( + TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2), + TypeCeiling(gpu_product=A100.gpu_product, accelerators=2), + ), + shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + ) + report = two_lanes(capacity=capacity).report + assert [(use.scope, use.gpu_product, use.ceiling, use.accelerators, use.lanes) for use in report.capacity] == [ + (CapacityScope.TOTAL, None, 4, 4, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.TYPE, A100.gpu_product, 2, 2, ("org-b-a100-80gb",)), + (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, ("org-a-l4",)), + (CapacityScope.SHARED_QUOTA, None, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + ] + assert not any(use.exceeded for use in report.capacity) + assert report.to_dict()["capacity"][3] == { + "scope": "shared-quota", + "ceiling": 6, + "accelerators": 6, + "heldElsewhere": 2, + "lanes": ["org-a-l4", "org-b-a100-80gb"], + "exceeded": False, + } + + +@pytest.mark.parametrize( + ("capacity", "message"), + [ + ( + Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)), + "the envelope maxima of org-a-l4 hold 2 NVIDIA-L4 accelerators, above the type ceiling of 1", + ), + ( + Capacity(total=3), + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, above the total ceiling of 3", + ), + ( + Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "which with 3 held elsewhere exceed the shared quota of 6", + ), + ], +) +def test_a_plan_whose_envelope_maxima_exceed_a_ceiling_is_refused_and_says_why(capacity, message): + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert isinstance(caught.value, PlannerInputError) + assert str(caught.value) == message + (exceeded,) = caught.value.exceeded + assert exceeded.exceeded + refused = caught.value.result + assert [lane.name for lane in refused.plan.lanes] == ["org-a-l4", "org-b-a100-80gb"] + assert [use["exceeded"] for use in refused.report.to_dict()["capacity"]] == [True] + + +def test_every_exceeded_ceiling_is_named(): + capacity = Capacity(total=3, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL, CapacityScope.TYPE] + assert str(caught.value).count("the envelope maxima") == 2 + + +def test_a_candidate_list_lane_counts_against_every_member_type(): + capacity = Capacity( + types=( + TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1), + TypeCeiling(gpu_product=A100.gpu_product, accelerators=1), + ) + ) + result = run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + assert [use.accelerators for use in result.report.capacity] == [1, 1] + capacity = Capacity(types=(TypeCeiling(gpu_product=A100.gpu_product, accelerators=0),)) + with pytest.raises(CapacityExceededError, match="hold 1 NVIDIA-A100-SXM4-80GB accelerators"): + run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + + +def test_each_container_holds_its_gpu_count(): + result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0) + assert [use.accelerators for use in result.report.capacity] == [2] + with pytest.raises(CapacityExceededError, match="hold 2 accelerators"): + run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1)) + + +def test_a_zero_floor_lane_holds_nothing_until_its_envelope_may_raise_it(): + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=0)) + assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + assert [(use.accelerators, use.lanes) for use in result.report.capacity] == [(0, ())] + + +def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): + receipts, boots = [receipt()], [attempt(boot=240.0)] + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) + assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] + capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) + assert bounded.plan == free.plan + assert bounded.report.come_alive_levers == () + assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 + with pytest.raises(CapacityExceededError): + run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) + + +def test_without_capacity_the_report_lists_no_ceiling(): + report = two_lanes().report + assert report.capacity == () + assert report.to_dict()["capacity"] == [] + + +CELLS = (L4, A100, H100) +FIELDS = {"comeAliveP95S": "come_alive_p95_s", "warmP99Ms": "warm_p99_ms"} + + +@pytest.mark.parametrize("seed", range(120)) +def test_random_estates_respect_placement_evidence_and_capacity(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = [f"org/model-{index}" for index in range(rng.randint(1, 4))] + service_levels = ServiceLevels( + identities=tuple( + IdentityServiceLevel( + identity=identity, + level=ServiceLevel( + come_alive_p95_s=rng.choice([None, 0, 10, 30, 120, 600]), + warm_p99_ms=rng.choice([150, 250, 400]), + ), + ) + for identity in identities + ) + ) + receipts = [ + receipt( + identity, cell, concurrency=rng.choice([4, 8]), rps=rng.choice([0.5, 5.0]), p99=rng.choice([100.0, 300.0]) + ) + for identity in identities + for cell in CELLS + if rng.random() < 0.6 + ] + boots = [] + for identity in identities: + for cell in CELLS: + for warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD): + for index in range(rng.randint(0, 3)): + outcome = rng.choice( + [BootOutcome.SUCCEEDED, BootOutcome.SUCCEEDED, BootOutcome.FAILED, BootOutcome.UNPLACED] + ) + boots.append( + attempt( + identity, + cell, + warmth=warmth, + boot=rng.choice([5.0, 20.0, 90.0]), + wait=rng.choice([None, 0.0, 15.0, 120.0]), + outcome=outcome, + borrowed=rng.random() < 0.2, + index=index, + ) + ) + for index in range(rng.randint(0, 6)): + boots.append(unplaced(rng.choice(CELLS), index=index)) + hold = rng.choice([0, 20, 60]) + capacity = rng.choice( + [ + None, + Capacity(total=rng.randint(0, 8)), + Capacity( + types=tuple(TypeCeiling(gpu_product=cell.gpu_product, accelerators=rng.randint(0, 4)) for cell in CELLS) + ), + Capacity(shared_quota=SharedQuota(accelerators=8, held_elsewhere=rng.randint(0, 8))), + ] + ) + overrides = {"hold_budget_s": hold, "buffer_containers": rng.randint(0, 1)} + hourly = { + identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} + for identity in identities + } + + def attempt_plan(current=None, order=False, given_levels=service_levels): + given_receipts, given_boots = list(receipts), list(boots) + if order: + rng.shuffle(given_receipts) + given_boots.reverse() + try: + return run( + given_levels, + given_receipts, + given_boots, + hourly=hourly, + current=current, + capacity=capacity, + **overrides, + ), None + except CapacityExceededError as error: + return error.result, error + + def relaxed(identity, blockers): + bounds = {} + for blocker in blockers: + if blocker.field == "placementFailureRate": + name, value = "come_alive_p95_s", hold + 1 + elif blocker.measured is not None: + name, value = FIELDS[blocker.field], max(1, math.ceil(blocker.measured)) + else: + continue + bounds[name] = max(value, bounds.get(name, 0)) + return ServiceLevels( + identities=tuple( + replace(entry, level=replace(entry.level, **bounds)) if entry.identity == identity else entry + for entry in service_levels.identities + ) + ) + + def fall(identity, blockers): + again, again_error = attempt_plan(current=result.plan, given_levels=relaxed(identity, blockers)) + return 0.0 if again_error is not None else max(0.0, result.report.monthly_cost - again.report.monthly_cost) + + result, error = attempt_plan() + attempts: dict[str, list[BootEvidence]] = {} + for row in boots: + if row.warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD): + attempts.setdefault(row.cell.gpu_product, []).append(row) + + def failure_rate(product): + rows = attempts.get(product, []) + own = [row for row in rows if not row.borrowed] or rows + return sum(row.outcome is BootOutcome.UNPLACED for row in own) / len(own) if own else 0.0 + + by_uri = {document.source.uri: document for document in receipts} + levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} + for entry in result.report.identities: + if entry.chosen is None: + continue + chosen = entry.chosen + level = levels_by_identity[entry.identity] + assert chosen.blocked_by == () + tight = level.come_alive_p95_s is not None and level.come_alive_p95_s <= hold + if tight and not chosen.fallback_cells: + assert failure_rate(chosen.cell.gpu_product) <= 0.05 + if chosen.fallback_cells: + assert chosen.warmth is Warmth.PINNED + assert tight + assert failure_rate(chosen.cell.gpu_product) > 0.05 + (second,) = chosen.fallback_cells + assert second.gpu_product != chosen.cell.gpu_product + assert RATES[second] >= RATES[chosen.cell] + assert any( + document.identity == entry.identity + and document.cell == second + and document.concurrency == chosen.concurrency + and document.latency.p99_ms <= level.warm_p99_ms + for document in receipts + ) + assert by_uri[chosen.receipt.uri].latency.p99_ms <= level.warm_p99_ms + if chosen.warmth is not Warmth.PINNED: + assert level.come_alive_p95_s is None or chosen.come_alive_s <= level.come_alive_p95_s + if chosen.placement_wait_s is not None: + assert chosen.come_alive_s >= chosen.placement_wait_s + for scarcity in result.report.scarcity: + chosen = result.report.identity(scarcity.identity).chosen + if chosen is not None: + assert scarcity.monthly_cost < chosen.monthly_cost + if scarcity.reason is ScarcityReason.UNRELIABLE_TYPE: + assert scarcity.placement_failure_rate > 0.05 + for use in result.report.capacity: + lanes = [ + planned + for planned in result.plan.lanes + if use.scope is not CapacityScope.TYPE or use.gpu_product in {cell.gpu_product for cell in planned.cells} + ] + held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) + elsewhere = use.held_elsewhere or 0 + assert use.accelerators == held + elsewhere + assert (error is not None) is any(use.exceeded for use in result.report.capacity) + if error is not None: + assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) + reordered, _ = attempt_plan(order=True) + assert rendered(reordered) == rendered(result) + if error is None: + for entry in result.report.identities: + if entry.next_cheaper is not None: + assert entry.next_cheaper.saving == pytest.approx( + fall(entry.identity, entry.next_cheaper.blocked_by), abs=0.01 + ) + for lever in result.report.come_alive_levers: + measured = Blocker(field="comeAliveP95S", bound=None, measured=lever.permitted_bound_s) + assert lever.saving_at_permitted_bound == pytest.approx(fall(lever.identity, [measured]), abs=0.01) + again, again_error = attempt_plan(current=result.plan) + assert again_error is None + assert again.plan == result.plan + assert again.report.unchanged diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 0141707b1..2bcbd0718 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -505,11 +505,23 @@ def test_twenty_or_more_boots_stand_for_the_come_alive_at_their_95th_percentile( assert rendered(run(level(come_alive_p95_s=600), [receipt()], boots[::-1], prices=L4_ONLY)) == rendered(result) -def test_a_failed_boot_withdraws_the_warmth_however_many_boots_succeeded(): - boots = [boot(seconds=50.0, uri=f"boots/{index}.json") for index in range(30)] - boots.append(boot(seconds=0.0, outcome=BootOutcome.FAILED, uri="boots/failed.json")) +@pytest.mark.parametrize( + ("succeeded", "failed", "come_alive"), + [(18, 1, None), (19, 1, 50.0), (30, 1, 50.0), (29, 2, None), (38, 2, 50.0), (37, 3, None)], +) +def test_a_failed_boot_ranks_above_every_success_at_the_95th_percentile(succeeded, failed, come_alive): + boots = [boot(seconds=50.0, uri=f"boots/{index}.json") for index in range(succeeded)] + boots.extend( + boot(seconds=0.0, outcome=BootOutcome.FAILED, uri=f"boots/failed-{index}.json") for index in range(failed) + ) result = run(level(come_alive_p95_s=600), [receipt()], boots, prices=L4_ONLY) - assert result.report.identity(IDENTITY).chosen.warmth is Warmth.PINNED + chosen = result.report.identity(IDENTITY).chosen + if come_alive is None: + assert chosen.warmth is Warmth.PINNED + (request,) = result.report.measurement_requests + assert request.reason is RequestReason.FAILED + else: + assert (chosen.warmth, chosen.come_alive_s) == (Warmth.COLD_WITH_BOUND, come_alive) def test_boot_seconds_render_the_same_whatever_their_number_type(): @@ -1456,6 +1468,7 @@ def test_the_region_multiplier_scales_every_price(): ({"fallback_floor": -1}, "fallbackFloor"), ({"cooldown_s": -1}, "cooldownS"), ({"buffer_containers": -1}, "bufferContainers"), + ({"hold_budget_s": -1}, "holdBudgetS"), ({"hours_per_month": 0}, "hoursPerMonth"), ({"hours_per_month": math.inf}, "hoursPerMonth"), ({"queue_pool": "two words"}, "queuePool"), @@ -1478,6 +1491,7 @@ def test_parameters_echo_into_the_report(): "fallbackFloor": 2, "cooldownS": 900, "bufferContainers": 0, + "holdBudgetS": 0, "hoursPerMonth": 730.0, "queuePool": "default", "bundle": "default", diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index c05a873c3..841c8eead 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -10,6 +10,7 @@ import pytest from sie_placement.schema import ( BootEvidence, + Capacity, Cell, ContainerShape, Demand, @@ -27,6 +28,8 @@ SchemaError, ServiceLevel, ServiceLevels, + SharedQuota, + TypeCeiling, Warmth, ) @@ -39,6 +42,7 @@ ("boot-evidence.json", BootEvidence.from_dict), ("price-list.json", PriceList.from_dict), ("demand.json", Demand.from_dict), + ("capacity.json", Capacity.from_dict), ("plan.json", Plan.from_dict), ] @@ -354,6 +358,100 @@ def test_boot_evidence_borrowed_is_a_strict_boolean(value, described): assert str(schema_error(BootEvidence.from_dict, data)) == f"$.borrowed: expected a boolean, got {described}" +def test_boot_evidence_records_an_unplaced_attempt_without_a_boot(): + data = load("boot-evidence.json") + data["outcome"] = "unplaced" + del data["bootSeconds"] + evidence = BootEvidence.from_dict(data) + assert (evidence.outcome.value, evidence.boot_seconds, evidence.placement_wait_s) == ("unplaced", None, 41.2) + assert evidence.to_dict() == data + del data["placementWaitS"] + assert BootEvidence.from_dict(data).to_dict() == data + data["bootSeconds"] = 30 + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.bootSeconds: must be omitted for an unplaced attempt, which never booted" + ) + + +@pytest.mark.parametrize("outcome", ["succeeded", "failed", "timed-out"]) +def test_boot_evidence_of_a_placed_attempt_records_its_boot(outcome): + data = load("boot-evidence.json") + data["outcome"] = outcome + del data["bootSeconds"] + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.bootSeconds: is required for an attempt that was placed" + ) + + +def test_boot_evidence_placement_waits_are_optional_and_non_negative(): + data = load("boot-evidence.json") + del data["placementWaitS"] + evidence = BootEvidence.from_dict(data) + assert evidence.placement_wait_s is None + assert "placementWaitS" not in evidence.to_dict() + data["placementWaitS"] = -1 + assert str(schema_error(BootEvidence.from_dict, data)) == "$.placementWaitS: must be at least 0, got -1" + + +def test_a_lazy_resident_load_runs_on_a_placed_container(): + data = load("boot-evidence.json") + data["warmth"] = "lazy-resident" + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.placementWaitS: must be omitted for a lazy-resident load, which runs on a placed container" + ) + del data["placementWaitS"] + assert BootEvidence.from_dict(data).warmth is Warmth.LAZY_RESIDENT + data["outcome"] = "unplaced" + del data["bootSeconds"] + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.outcome: a lazy-resident load runs on a placed container and is never unplaced" + ) + + +def test_capacity_declares_a_total_type_ceilings_or_a_shared_quota(): + assert str(schema_error(Capacity.from_dict, {"kind": "placement-capacity", "schemaVersion": 1})) == ( + "$: must declare total, types or sharedQuota" + ) + assert Capacity(total=0).to_dict() == {"kind": "placement-capacity", "schemaVersion": 1, "total": 0} + quota = Capacity.from_dict( + {"kind": "placement-capacity", "schemaVersion": 1, "sharedQuota": {"accelerators": 4}} + ).shared_quota + assert quota == SharedQuota(accelerators=4, held_elsewhere=0) + assert quota.to_dict() == {"accelerators": 4, "heldElsewhere": 0} + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda data: data.update(total=-1), "$.total: must be at least 0, got -1"), + (lambda data: data["types"][1].update(accelerators=-2), "$.types[1].accelerators: must be at least 0, got -2"), + ( + lambda data: data["types"][1].update(gpuProduct="NVIDIA-L4"), + "$.types[1].gpuProduct: duplicates 'NVIDIA-L4'", + ), + (lambda data: data["types"][0].update(gpuProduct="NVIDIA L4"), "$.types[0].gpuProduct: must be a non-empty"), + ( + lambda data: data["sharedQuota"].update(heldElsewhere=-1), + "$.sharedQuota.heldElsewhere: must be at least 0, got -1", + ), + (lambda data: data["sharedQuota"].pop("accelerators"), "$.sharedQuota.accelerators: is required"), + (lambda data: data["types"][0].update(max=3), "$.types[0].max: is not a field of this object"), + ], +) +def test_capacity_ceilings_are_validated(mutate, message): + data = load("capacity.json") + mutate(data) + assert str(schema_error(Capacity.from_dict, data)).startswith(message) + + +def test_capacity_built_in_code_names_type_ceilings_by_gpu_product(): + capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + assert Capacity.from_dict(json.loads(json.dumps(capacity.to_dict()))) == capacity + with pytest.raises(SchemaError) as caught: + TypeCeiling(gpu_product="", accelerators=1) + assert caught.value.path == "gpuProduct" + + def test_price_lists_validate_currency_rates_and_names(): data = load("price-list.json") data["currency"] = "usd" @@ -523,6 +621,40 @@ def test_plan_accepts_every_zero_floor_warmth(warmth): assert Plan.from_dict(data).identities[2].warmth.value == warmth +def test_plan_lanes_name_fallback_cells_after_their_cell(): + data = load("plan.json") + lane = Plan.from_dict(data).lanes[1] + assert lane.cells == ( + Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB"), + Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3"), + ) + assert "fallbackCells" not in Plan.from_dict(data).lanes[0].to_dict() + assert Plan.from_dict(data).lanes[0].cells == (Plan.from_dict(data).lanes[0].cell,) + + +@pytest.mark.parametrize( + ("cells", "message"), + [ + ( + [{"machineProfile": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB"}], + "$.lanes[1].fallbackCells[0]: repeats 'a100-80gb' with 'NVIDIA-A100-SXM4-80GB', already a cell of this lane", + ), + ( + [ + {"machineProfile": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3"}, + {"machineProfile": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3"}, + ], + "$.lanes[1].fallbackCells[1]: repeats 'h100' with 'NVIDIA-H100-80GB-HBM3', already a cell of this lane", + ), + ({"machineProfile": "h100"}, "$.lanes[1].fallbackCells: expected an array, got an object"), + ], +) +def test_plan_lane_fallback_cells_are_distinct_from_each_other_and_the_cell(cells, message): + data = load("plan.json") + data["lanes"][1]["fallbackCells"] = cells + assert str(schema_error(Plan.from_dict, data)) == message + + def test_plan_lane_names_and_identities_are_unique(): data = load("plan.json") data["lanes"][1]["name"] = "default-l4-default" From c936cadfe3e11d5b970b46e9843ac9fab63dc37e Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 17:58:19 +0100 Subject: [PATCH 13/40] feat(placement): pack several models onto one GPU and price shared lanes jointly One card can hold several models: some pinned and always resident, the rest loaded on demand. The planner now forms such lanes from measured evidence and prices a shared lane once for all its residents. - Receipts record footprintMiB, workingMemoryMiB and coResidents, boot records the footprintMiB of a load that succeeded, and price-list profiles gpuMemoryMiB. A lane fits when its pinned footprints, its largest lazy footprint and its largest working memory at the lane's concurrency come to at most 80% of the card. Only single-card lanes without fallback cells hold several identities. - A pinned identity beside other pinned identities is judged by a co-residency receipt whose coResidents include them, and its demand is priced at that receipt. A receipt measured beside more residents stands for fewer. A current shared lane without one is kept and asks for it. - Lazy-resident options on current host lanes need the memory check, and a footprint or working memory that is not on file is a measurement request. - A packing pass visits every identity alone on its lane in decreasing floor cost and tries lanes that pin a never-cold identity first, then in decreasing footprint in use. It keeps a move only when the plan total falls and every resident's service level holds, and a move that lacks footprints, a load or a co-residency receipt becomes a measurement request bounded by that fall. - Shared lanes are priced jointly: identities that left a lane their neighbours keep return to it, alone or together, whenever that does not raise the plan total, and an identity whose co-pinned neighbours change in the run is decided again beside the ones it is left with. Loosening a bound can no longer raise the plan total because a pinned identity left a shared lane; the levers test that pinned that defect now asserts the identity stays. - Generation identities are never packed. --- packages/sie_placement/README.md | 143 +++- .../src/sie_placement/planner.py | 590 ++++++++++++++- .../sie_placement/src/sie_placement/report.py | 10 + .../sie_placement/src/sie_placement/schema.py | 49 +- .../fixtures/planner/expected-report.json | 14 +- .../tests/fixtures/planner/price-list.json | 4 + .../tests/fixtures/planner/receipts.json | 44 ++ .../sie_placement/tests/test_availability.py | 11 +- packages/sie_placement/tests/test_levers.py | 20 +- packages/sie_placement/tests/test_packing.py | 694 ++++++++++++++++++ packages/sie_placement/tests/test_planner.py | 139 ++-- packages/sie_placement/tests/test_schema.py | 73 ++ 12 files changed, 1668 insertions(+), 123 deletions(-) create mode 100644 packages/sie_placement/tests/test_packing.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 304094fcf..494d2cfc7 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -25,9 +25,9 @@ name the offending field by its JSON path, for example | Kind | Type | Contents | | --- | --- | --- | | `placement-service-levels` | `ServiceLevels` | Optional named `tiers` and the `identities` that declare or inherit service levels | -| `placement-receipt` | `Receipt` | One throughput and latency measurement of one identity on one cell, container shape, concurrency and request shape | -| `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape | -| `placement-price-list` | `PriceList` | Accelerator, core and memory rates per machine profile, and the region multiplier per placement | +| `placement-receipt` | `Receipt` | One throughput and latency measurement of one identity on one cell, container shape, concurrency and request shape, alone or beside named co-residents, with the GPU memory the identity holds | +| `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape, with the GPU memory a successful one left in use | +| `placement-price-list` | `PriceList` | Accelerator, core and memory rates and the card memory per machine profile, and the region multiplier per placement | | `placement-demand` | `Demand` | Hourly request counts per identity and operation over a UTC window | | `placement-capacity` | `Capacity` | The accelerators a plan may hold: in total, per GPU product, and in a quota it shares with other environments | | `placement-plan` | `Plan` | Lanes with their cell, fallback cells, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | @@ -120,6 +120,40 @@ A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane with fallback cells counts against the ceiling of every product it names, and once against `total`. +## Several models on one GPU + +A lane can hold several identities on one card: some `pinned`, always +resident, and the rest `lazy-resident`, loaded on demand and evicted least +recently used. The planner decides this from measurements only: + +- `footprintMiB`, on a receipt or on a boot record that succeeded, is the GPU + memory the identity holds once loaded: the memory in use after its load less + the memory in use before it. The planner takes the largest an identity + records on a cell. +- `workingMemoryMiB`, on a receipt, is the GPU memory its run needed at the + receipt's concurrency beyond the footprint. The planner takes the largest an + identity records at a cell, container shape and concurrency. +- `coResidents`, on a receipt, names the identities pinned and loaded on the + same container while it was measured, which makes it a co-residency receipt. + A receipt without `coResidents` measured the identity alone and is a cell; + a co-residency receipt is not. +- `gpuMemoryMiB`, on a price-list machine profile, is the memory of one of its + cards. + +A lane fits its residents when the footprints of its pinned identities, the +largest footprint of its lazy residents and the largest working memory any +resident needs at the lane's concurrency come to at most 80% of `gpuMemoryMiB`. +The planner puts an identity beside another only on a lane that fits them, of +one card per container and without fallback cells, on a profile that states +`gpuMemoryMiB`. A pinned identity beside other pinned identities is judged by a +co-residency receipt at the lane's cell, container shape and concurrency rather +than by a receipt of it alone: one whose `coResidents` include every other +identity pinned on the lane, since a receipt measured beside more residents +stands for fewer. Its warm bounds are checked and its demand is priced at that +receipt. The planner never places a generation identity, one that declares +`ttftP99Ms` and `itlP99Ms` or has a generate receipt, beside another identity, +nor another identity on a lane that hosts one. + ## Envelope A plan lane's envelope bounds what an online tuner may change without a new @@ -157,11 +191,7 @@ and its report. same bytes in whatever order the receipts, boot evidence, identities and demand series arrive. An identity that stays on its current placement keeps its current lane as it is, so a run whose `current_plan` already holds every -choice, such as its own previous plan, reports no change. The exceptions are -lazy residents that join a host lane one run later: a lane the previous run -created for an identity whose `comeAliveP95S` is `0` hosts lazy residents only -from the run after it, and an identity that left a host lane it was pinned on -may rejoin that lane as a lazy resident on the next run. +choice, such as its own previous plan, reports no change. | Parameter | Meaning | | --- | --- | @@ -216,7 +246,12 @@ document: since the host's container is already placed. The option costs what the identity's demand needs above the host lane's floor, plus the floor itself on a lane where no such identity stays pinned, since no other identity pays - it. A lane with fallback cells hosts no lazy resident. + it. The host must fit the identity beside the residents of its current + lane, as [Several models on one GPU](#several-models-on-one-gpu) states; + where a footprint or a working memory is not on file the option is a + measurement request for it. A lane with fallback cells, or one that hosts a + generation identity, hosts no lazy resident, and a generation identity is + never one. 5. A GPU product is unreliable when more than 5% of the cold attempts on it were `unplaced`, counting the environment's own attempts in place of borrowed ones. For an identity whose `comeAliveP95S` is at most @@ -234,21 +269,23 @@ The current placement is the identity's current lane when that lane has the option's cell and fallback cells, container shape and concurrency, the identity's warmth is the option's, and the lane's `queuePool` and `bundle` are the ones the parameters give the identity; a `lazy-resident` option must also be -hosted by that lane. A current placement that no option matches lacks evidence: -a receipt at its lane's cell, container shape and concurrency, a boot for a cold -warmth, or a load for a lazy resident. Unless it is known to miss a bound, -through a receipt at the lane that misses a warm bound, a measured come-alive -over the bound, a boot or load at the lane that did not succeed, a cold or -`lazy-resident` warmth under a bound of `0`, or an unreliable product without a -fallback cell on another product under a bound `hold_budget_s` fits, the -identity keeps that placement and leaves it only for an unblocked option at -least a cent cheaper than the placement. The placement is priced as an option of -its warmth at the lane's fastest receipt, or without one at its floor alone: one -container all month for `pinned`, for `lazy-resident` nothing beyond the lane's -floor, or the floor itself where no identity whose `comeAliveP95S` is `0` stays -pinned on the lane, and the burst tails for a cold warmth. What the placement -lacks becomes a measurement request. Such a placement on a lane the price list -cannot price has no cost to compare, so the identity keeps it. +hosted by that lane, and a `pinned` option on a lane where other identities are +pinned must stand on a co-residency receipt beside them. A current placement +that no option matches lacks evidence: a receipt at its lane's cell, container +shape and concurrency, a co-residency receipt beside the identities pinned with +it, a boot for a cold warmth, or a load for a lazy resident. Unless it is known +to miss a bound, through a receipt at the lane that misses a warm bound, a +measured come-alive over the bound, a boot or load at the lane that did not +succeed, a cold or `lazy-resident` warmth under a bound of `0`, or an unreliable +product without a fallback cell on another product under a bound `hold_budget_s` +fits, the identity keeps that placement and leaves it only for an unblocked +option at least a cent cheaper than the placement. The placement is priced as an +option of its warmth at the lane's fastest receipt, or without one at its floor +alone: one container all month for `pinned`, for `lazy-resident` nothing beyond +the lane's floor, or the floor itself where no identity whose `comeAliveP95S` is +`0` stays pinned on the lane, and the burst tails for a cold warmth. What the +placement lacks becomes a measurement request. Such a placement on a lane the +price list cannot price has no cost to compare, so the identity keeps it. An identity that stays on its current placement keeps that lane exactly as it is, beside whoever else the lane hosts: its cell, container shape, @@ -260,6 +297,36 @@ the identity added when that name is taken. The planner does not select current lane, capped by the reviewed ceiling, or stays out of the plan when it has none. +### Sharing a lane + +Once every identity has an option, the planner prices the plan as a whole and +changes it in the steps below. A floor is paid once per lane, so an option an +identity is offered alone can look cheaper than the lane it shares while the +plan total does not fall. + +1. A pinned identity whose co-pinned neighbours change in this run is decided + again beside the neighbours it is left with, until no identity's + neighbours change. +2. Identities that left a current lane return to it, one at a time and then + together, wherever their return does not raise the plan total. An identity + whose current placement is known to miss a bound does not return. +3. Packing. The planner visits every placed identity that is alone on its lane + and is not a generation identity, in decreasing order of its lane's floor + cost, ties broken by identity. For each it tries the other lanes of the plan + on its queue pool and bundle that keep a floor and host a pinned identity, + where it has a receipt at the lane's cell, container shape and concurrency: + first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, + since nothing moves that identity, then in decreasing order of the footprint + already on them, then by name. On the first lane where a move fits and lowers + the plan total by at least a cent, the identity joins it, as a lazy resident + when its bound admits its measured load onto the lane and selection did not + already offer it the lane, otherwise pinned, which needs every pinned + identity's co-residency receipt beside the others to meet its warm bounds. A + move that would lower the total but lacks a footprint, a working memory, a + load or a co-residency receipt is not made: it is a measurement request whose + `savingBound` is that fall. +4. Step 1 runs again for the lanes packing changed. + ### Pricing A container costs `(acceleratorPerHour * gpuCount + corePerHour * cpuCores + @@ -309,8 +376,10 @@ The report renders as a `placement-report` document: demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or `borrowed-boot-evidence`), the `placementWaitS` part of a cold come-alive - where a placement wait was measured, the `fallbackCells` of a candidate list - and, for a lazy resident, its host `lane`, the peak containers its demand + where a placement wait was measured, the `fallbackCells` of a candidate list, + the `coResidents` of the co-residency receipt a co-pinned identity is judged + by and, for a resident that joins another identity's lane, its `lane`, the + peak containers its demand needs, and `nextCheaper`: the nearest option cheaper than the choice, or than a kept identity's current placement, that a service level blocks, the bounds that block it, and `saving`, the fall in the plan total found by planning @@ -342,7 +411,9 @@ The report renders as a `placement-report` document: and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, and `unpricedIdentities` lists residents whose demand has no receipt to price - it on the lane. + it on the lane. A lane that hosts more than one identity names the GPU memory + its residents need, `memoryMiB`, and the 80% of its card's memory they may + use, `memoryLimitMiB`, where both are known. - `sharedChartLanes`: every chart lane, by `queuePool`, `machineProfile` and `bundle`, that more than one plan lane names. - `measurementRequests`: options that lack evidence, ranked by the saving they @@ -354,12 +425,18 @@ The report renders as a `placement-report` document: reason `failed`, or `unplaced` when no attempt got a container or the product's placement wait falls on an unplaced attempt, and a host lane without the identity's load asks for `lazy-resident` boot evidence on it, bounded by - what that option would cost. Savings are measured against the choice, or - against a kept identity's current placement. A request that cannot save is - dropped, and the requests of an identity without a placement or a current one - carry no bound and rank first. The evidence a kept current placement lacks is - requested at its lane, named in `lane` with its `concurrency`, and also ranks - first without a bound; a profile it asks for there is not asked for again. + what that option would cost, with `footprint` also asked for where a + resident's footprint or working memory is not on file. A packing that lacks + evidence asks at the lane it would join for `boot-evidence`, `footprint` or + `co-residency-receipt`, with the `coResidents` the receipt is to be measured + beside, bounded by the fall in the plan total the packing would bring. A kept + identity pinned beside others asks at its lane for a `co-residency-receipt` + beside them. Savings are measured against the choice, or against a kept + identity's current placement. A request that cannot save is dropped, and the + requests of an identity without a placement or a current one carry no bound + and rank first. The evidence a kept current placement lacks is requested at + its lane, named in `lane` with its `concurrency`, and also ranks first without + a bound; a profile it asks for there is not asked for again. - `comeAliveLevers`: cold and `lazy-resident` warmths whose measured come-alive misses the bound on a cell that meets the warm bounds for less than the choice, or than a kept identity's current placement, listed only when diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index bf286ef09..54dc9296b 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -65,6 +65,7 @@ BOOT_PERCENTILE_MIN_ROWS = 20 PLACEMENT_FAILURE_RATE_LIMIT = 0.05 PLACEMENT_FAILURE_FIELD = "placementFailureRate" +MEMORY_HEADROOM_FRACTION = 0.8 COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") @@ -262,6 +263,7 @@ class _Option: blockers: tuple[Blocker, ...] current: bool host: str | None = None + host_owner: str | None = None fallbacks: tuple[Receipt, ...] = () boot_s: float | None = None placement_wait_s: float | None = None @@ -305,9 +307,32 @@ def summary(self, saving: float | None = None) -> Option: lane=self.host, fallback_cells=self.cells[1:], placement_wait_s=self.placement_wait_s, + co_residents=self.receipt.co_residents, ) +@dataclass(frozen=True, slots=True, kw_only=True) +class _Memory: + fits: bool | None + need: int | None = None + limit: float | None = None + + +_UNPACKABLE = _Memory(fits=False) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Layout: + plan: Plan + names: dict[str, str] + kept: dict[str, tuple[PlanIdentity, PlanLane]] + receipts: dict[str, Receipt | None] + owners: dict[str, str] + + +type _CoKey = tuple[str, Cell, ContainerShape, int] + + @dataclass(frozen=True, slots=True, kw_only=True) class _Choice: identity: str @@ -321,6 +346,7 @@ class _Choice: gaps: tuple[MeasurementRequest, ...] = () unpriced: bool = False withdrawn: tuple[_Option, ...] = () + beside: frozenset[str] = frozenset() @property def baseline(self) -> float | None: @@ -544,14 +570,42 @@ def __init__( self.current = { entry.identity: (entry, self.current_lanes[entry.lane]) for entry in current_plan.identities } + self.current_pinned: defaultdict[str, set[str]] = defaultdict(set) + self.current_lazy: defaultdict[str, set[str]] = defaultdict(set) + if current_plan is not None: + for entry in current_plan.identities: + if entry.warmth is Warmth.PINNED: + self.current_pinned[entry.lane].add(entry.identity) + elif entry.warmth is Warmth.LAZY_RESIDENT: + self.current_lazy[entry.lane].add(entry.identity) self.refusals: list[Refusal] = [] self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} + self.co_receipts: defaultdict[_CoKey, list[tuple[str, Receipt]]] = defaultdict(list) self.receipts = self._accept_receipts(receipts) self.placements = _placement_index( record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None ) - self.boots = _boot_index(self._accept_boot_evidence(boot_evidence), self.placements) + accepted = self._accept_boot_evidence(boot_evidence) + self.boots = _boot_index(accepted, self.placements) + self.footprints: dict[tuple[str, Cell], int] = {} + self.working: dict[tuple[str, Cell, ContainerShape, int], int] = {} + measured = [receipt for pairs in (*self.receipts.values(), *self.co_receipts.values()) for _, receipt in pairs] + for receipt in measured: + self._footprint(receipt.identity, receipt.cell, receipt.footprint_mib) + if receipt.working_memory_mib is not None: + key = (receipt.identity, receipt.cell, receipt.container_shape, receipt.concurrency) + self.working[key] = max(receipt.working_memory_mib, self.working.get(key, 0)) + for record in accepted: + self._footprint(record.identity, record.cell, record.footprint_mib) + self.generation = {identity for identity, level in self.levels.items() if level.is_generation} + self.generation.update( + receipt.identity for receipt in measured if receipt.request_shape.operation is Operation.GENERATE + ) + + def _footprint(self, identity: str, cell: Cell, footprint: int | None) -> None: + if footprint is not None: + self.footprints[(identity, cell)] = max(footprint, self.footprints.get((identity, cell), 0)) def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[str, Receipt]]]: accepted: defaultdict[str, list[tuple[str, Receipt]]] = defaultdict(list) @@ -562,9 +616,15 @@ def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[ reason = _refusal_reason(receipt.cell, self.prices) or _incomparable_reason(level, receipt) if reason is not None: self._refuse(RefusedRecord.RECEIPT, receipt.identity, receipt.cell, receipt.source, reason) - self.refused_profiles.add((receipt.identity, receipt.cell.machine_profile)) + if not receipt.co_residents: + self.refused_profiles.add((receipt.identity, receipt.cell.machine_profile)) + continue + order = _canonical(receipt.to_dict()) + if receipt.co_residents: + key = (receipt.identity, receipt.cell, receipt.container_shape, receipt.concurrency) + self.co_receipts[key].append((order, receipt)) continue - accepted[receipt.identity].append((_canonical(receipt.to_dict()), receipt)) + accepted[receipt.identity].append((order, receipt)) return accepted def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvidence]: @@ -582,11 +642,21 @@ def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvi def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) - def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: tuple[Cell, ...] = ()) -> bool: + def is_current( + self, + identity: str, + receipt: Receipt, + warmth: Warmth, + fallbacks: tuple[Cell, ...] = (), + beside: frozenset[str] | None = None, + ) -> bool: current = self.current.get(identity) if current is None: return False entry, lane = current + context = self.pinned_beside(identity) if beside is None else beside + if warmth is Warmth.PINNED and not _stands_beside(receipt, context): + return False return ( entry.warmth is warmth and lane.queue_pool == self.parameters.queue_pool @@ -595,6 +665,58 @@ def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: and _runs_on(receipt, lane) ) + def pinned_beside(self, identity: str) -> frozenset[str]: + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + return frozenset() + return frozenset(self.current_pinned[current[1].name] - {identity}) + + def co_receipt(self, identity: str, lane: PlanLane, beside: frozenset[str]) -> tuple[str, Receipt] | None: + found = [ + pair + for pair in self.co_receipts.get((identity, lane.cell, lane.container_shape, lane.concurrency), []) + if _stands_beside(pair[1], beside) + ] + return min( + found, + key=lambda pair: ( + len(pair[1].co_residents), + -_p99(pair[1]), + pair[1].throughput.requests_per_second, + pair[0], + ), + default=None, + ) + + def hosts_generation(self, identities: Iterable[str]) -> bool: + return any(identity in self.generation for identity in identities) + + def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> _Memory: + pinned, lazy = tuple(sorted(pinned)), tuple(sorted(lazy)) + price = self.prices.get(lane.cell.machine_profile) + residents = (*pinned, *lazy) + if ( + not residents + or price is None + or price.gpu_memory_mib is None + or lane.container_shape.gpu_count != 1 + or lane.fallback_cells + ): + return _UNPACKABLE + limit = price.gpu_memory_mib * MEMORY_HEADROOM_FRACTION + config = (lane.cell, lane.container_shape, lane.concurrency) + if any( + (identity, lane.cell) not in self.footprints or (identity, *config) not in self.working + for identity in residents + ): + return _Memory(fits=None, limit=limit) + need = ( + sum(self.footprints[(identity, lane.cell)] for identity in pinned) + + max((self.footprints[(identity, lane.cell)] for identity in lazy), default=0) + + max(self.working[(identity, *config)] for identity in residents) + ) + return _Memory(fits=need <= limit, need=need, limit=limit) + def hourly(self, receipt: Receipt) -> float: return container_hourly_cost( self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier @@ -667,6 +789,7 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: if (lazy if name == home else self.floor_paid(name, identity)) and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells + and not self.hosts_generation((*self.current_pinned[name], *self.current_lazy[name])) and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -707,12 +830,21 @@ def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> fl return hourly * hours def current_gaps( - self, identity: str, entry: PlanIdentity, lane: PlanLane, cost: float | None + self, + identity: str, + entry: PlanIdentity, + lane: PlanLane, + cost: float | None, + beside: frozenset[str] | None = None, ) -> tuple[MeasurementRequest, ...]: if cost is None: return () missing: list[tuple[Measurement, RequestReason]] = [] - if self.slowest_receipt(identity, lane) is None: + beside = self.pinned_beside(identity) if beside is None else beside + if beside: + if self.co_receipt(identity, lane, beside) is None: + missing.append((Measurement.CO_RESIDENCY_RECEIPT, RequestReason.UNMEASURED)) + elif self.slowest_receipt(identity, lane) is None: refused = (identity, lane.cell.machine_profile) in self.refused_profiles missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) if entry.warmth in BOOT_WARMTHS and (identity, lane.cell, lane.container_shape, entry.warmth) not in self.boots: @@ -729,11 +861,13 @@ def current_gaps( reason=reason, cost_bound=cost, saving_bound=None, + co_residents=tuple(sorted(beside)) if measurement is Measurement.CO_RESIDENCY_RECEIPT else (), ) for measurement, reason in missing ) - def choose(self, identity: str) -> _Choice: + def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice: + beside = self.pinned_beside(identity) if beside is None else beside level = self.levels[identity] requests = self.requests.get(identity, ()) options: list[_Option] = [] @@ -742,7 +876,7 @@ def choose(self, identity: str) -> _Choice: load_requests: dict[str, MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) - hosts = self.lazy_hosts(identity) + hosts = () if identity in self.generation else self.lazy_hosts(identity) for order, receipt in receipts: hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second @@ -759,7 +893,7 @@ def choose(self, identity: str) -> _Choice: come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, blockers=blocked, - current=self.is_current(identity, receipt, Warmth.PINNED), + current=self.is_current(identity, receipt, Warmth.PINNED, beside=beside), ) ) cold_cost = hourly * self.window.cold_container_hours(requests, rate) @@ -815,7 +949,18 @@ def choose(self, identity: str) -> _Choice: continue lazy_cost = hourly * self.resident_hours(host, identity, requests, rate) load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) - if load is None or load.seconds is None: + memory = self.memory( + host, + pinned=self.current_pinned[host.name] - {identity}, + lazy=self.current_lazy[host.name] | {identity}, + ) + if memory.fits is False: + continue + missing = ( + *(() if load is not None and load.seconds is not None else (Measurement.BOOT_EVIDENCE,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + if missing: known = load_requests.get(host.name) if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): load_requests[host.name] = MeasurementRequest( @@ -824,13 +969,19 @@ def choose(self, identity: str) -> _Choice: container_shape=host.container_shape, concurrency=host.concurrency, lane=host.name, - measure=(Measurement.BOOT_EVIDENCE,), + measure=missing, warmth=Warmth.LAZY_RESIDENT, - reason=RequestReason.UNMEASURED if load is None else RequestReason.FAILED, + reason=( + RequestReason.FAILED + if load is not None and load.seconds is None + else RequestReason.UNMEASURED + ), cost_bound=lazy_cost, saving_bound=None, ) continue + if load is None or load.seconds is None: + continue blocker = _come_alive_blocker(level, load.seconds) options.append( _Option( @@ -847,8 +998,9 @@ def choose(self, identity: str) -> _Choice: boot_s=load.boot_s, ) ) + options.extend(self.beside_options(identity, level, requests, current_class, beside)) if self.tight(level): - options.extend(self.candidate_lists(identity, level, receipts, requests, current_class)) + options.extend(self.candidate_lists(identity, level, receipts, requests, current_class, beside)) feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None held = None @@ -864,7 +1016,7 @@ def choose(self, identity: str) -> _Choice: if chosen is None: held = cost unpriced = cost is None - gaps = self.current_gaps(identity, entry, lane, cost) + gaps = self.current_gaps(identity, entry, lane, cost, beside) gapped = { (request.cell, request.container_shape, request.warmth) for request in gaps @@ -886,8 +1038,44 @@ def choose(self, identity: str) -> _Choice: gaps=gaps, unpriced=unpriced, withdrawn=tuple(withdrawn), + beside=beside, ) + def beside_options( + self, + identity: str, + level: ServiceLevel, + requests: Sequence[int], + current_class: Cell | None, + beside: frozenset[str], + ) -> list[_Option]: + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + return [] + lane = current[1] + if not beside and any(_runs_on(receipt, lane) for _, receipt in self.receipts.get(identity, [])): + return [] + found = self.co_receipt(identity, lane, beside) + if found is None or not self.is_current(identity, found[1], Warmth.PINNED, beside=beside): + return [] + order, receipt = found + return [ + _Option( + receipt=receipt, + order=order, + warmth=Warmth.PINNED, + cost=self.hourly(receipt) + * self.window.warm_container_hours(requests, receipt.throughput.requests_per_second), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=( + *self.warm_blockers(level, receipt, current_class), + *self.reliability_blockers(level, lane.cells), + ), + current=True, + ) + ] + def candidate_lists( self, identity: str, @@ -895,6 +1083,7 @@ def candidate_lists( receipts: Sequence[tuple[str, Receipt]], requests: Sequence[int], current_class: Cell | None, + beside: frozenset[str], ) -> list[_Option]: lists = [] for order, receipt in receipts: @@ -921,7 +1110,7 @@ def candidate_lists( come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, blockers=(), - current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,)), + current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,), beside), ) ) return lists @@ -929,7 +1118,9 @@ def candidate_lists( def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} measured.update( - request.cell.machine_profile for request in choice.gaps if Measurement.RECEIPT in request.measure + request.cell.machine_profile + for request in choice.gaps + if Measurement.RECEIPT in request.measure or Measurement.CO_RESIDENCY_RECEIPT in request.measure ) bound = choice.level.come_alive_p95_s tight = self.tight(choice.level) @@ -1254,15 +1445,12 @@ def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) - return window.above_floor_hours(requests, rate, lane.envelope.floor_min) return window.cold_container_hours(requests, rate) - def run(self, *, price_savings: bool = True) -> PlannerResult: - never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] - self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) - choices = [self.choose(identity) for identity in sorted(self.levels)] - placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} + def assemble(self, choices: Mapping[str, _Choice]) -> _Layout: + placed = {identity: choice.chosen for identity, choice in choices.items() if choice.chosen is not None} kept = { - choice.identity: self.current[choice.identity] - for choice in choices - if choice.chosen is None and choice.identity in self.current + identity: self.current[identity] + for identity, choice in choices.items() + if choice.chosen is None and identity in self.current } staying = {identity: self.current[identity][1].name for identity, option in placed.items() if option.current} joining = { @@ -1270,13 +1458,19 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: for identity, option in placed.items() if (host := option.host) is not None and identity not in staying } + riding = { + identity: owner + for identity, option in placed.items() + if (owner := option.host_owner) is not None and identity not in staying + } reserved = {lane.name for _, lane in kept.values()} | set(staying.values()) | set(joining.values()) moving = { identity: option for identity, option in placed.items() - if identity not in staying and identity not in joining + if identity not in staying and identity not in joining and identity not in riding } names = staying | joining | self.lane_names(moving, reserved) + names.update((identity, names[owner]) for identity, owner in riding.items()) carried = [lane for _, lane in kept.values()] carried.extend(self.current_lanes[name] for name in (*staying.values(), *joining.values())) lanes = {lane.name: self.within_ceiling(lane) for lane in carried} @@ -1287,17 +1481,334 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: PlanIdentity(identity=identity, lane=names[identity], warmth=option.warmth) for identity, option in placed.items() ) - new_plan = Plan( + plan = Plan( placement=self.parameters.placement, lanes=tuple(lanes[name] for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) + return _Layout( + plan=plan, + names=names, + kept=kept, + receipts=receipts, + owners={names[identity]: identity for identity in moving}, + ) + + def total(self, layout: _Layout) -> float: + lane_reports, _ = self._pricing(layout.plan, layout.receipts) + return sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) + + def stay(self, choice: _Choice) -> _Choice | None: + identity = choice.identity + current = self.keepable(identity) + if current is None: + return None + standing = [option for option in choice.options if option.current] + feasible = [option for option in standing if option.feasible] + if feasible: + return replace(choice, chosen=min(feasible, key=lambda option: option.rank)) + entry, lane = current + if standing or self.kept_blockers(identity, entry, lane, self.slowest_receipt(identity, lane)): + return None + cost = self.current_cost(identity, entry, lane) + gaps = self.current_gaps(identity, entry, lane, cost, choice.beside) + gapped = { + (request.cell, request.container_shape, request.warmth) + for request in gaps + if Measurement.BOOT_EVIDENCE in request.measure + } + return replace( + choice, + chosen=None, + held=cost, + unpriced=cost is None, + gaps=gaps, + boot_requests=tuple( + request + for request in choice.boot_requests + if (request.cell, request.container_shape, request.warmth) not in gapped + ), + ) + + @staticmethod + def stays(identity: str, choice: _Choice, layout: _Layout) -> bool: + if choice.chosen is None: + return identity in layout.kept + return choice.chosen.current + + def stay_beside(self, choices: dict[str, _Choice], originals: Mapping[str, _Choice]) -> dict[str, _Choice]: + layout = self.assemble(choices) + total = self.total(layout) + left: defaultdict[str, dict[str, _Choice]] = defaultdict(dict) + for identity in sorted(choices): + if identity not in self.current or self.stays(identity, choices[identity], layout): + continue + stay = self.stay(originals[identity]) + if stay is not None: + left[self.current[identity][1].name][identity] = stay + for lane in sorted(left): + movers = left[lane] + groups = [{identity: stay} for identity, stay in movers.items()] + if len(movers) > 1: + groups.append(movers) + for group in groups: + trial = self.settle({**choices, **group}) + trial_layout = self.assemble(trial) + trial_total = self.total(trial_layout) + if round(trial_total - total, 2) <= 0: + choices, layout, total = trial, trial_layout, trial_total + return choices + + def riding_cost(self, lane: PlanLane, identity: str, receipt: Receipt) -> float: + window = replace(self.window, ceiling=lane.max_containers) + requests = self.requests.get(identity, ()) + rate = receipt.throughput.requests_per_second + return self.lane_hourly(lane) * window.above_floor_hours(requests, rate, lane.envelope.floor_min) + + def in_use(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> int: + pinned_use = sum(self.footprints.get((identity, lane.cell), 0) for identity in pinned) + return pinned_use + max((self.footprints.get((identity, lane.cell), 0) for identity in lazy), default=0) + + def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, frozenset[str], frozenset[str]]]: + pinned: defaultdict[str, set[str]] = defaultdict(set) + lazy: defaultdict[str, set[str]] = defaultdict(set) + for entry in layout.plan.identities: + (pinned if entry.warmth is Warmth.PINNED else lazy)[entry.lane].add(entry.identity) + targets = [ + (lane, frozenset(pinned[lane.name]), frozenset(lazy[lane.name])) + for lane in layout.plan.lanes + if lane.name != layout.names[identity] + and lane.envelope.floor_min >= PINNED_FLOOR + and not lane.fallback_cells + and lane.container_shape.gpu_count == 1 + and lane.queue_pool == self.parameters.queue_pool + and lane.bundle == self.parameters.bundle_of(identity) + and pinned[lane.name] + and not self.hosts_generation((*pinned[lane.name], *lazy[lane.name])) + and self.slowest_receipt(identity, lane) is not None + ] + return sorted( + targets, + key=lambda target: ( + not any(self.levels[resident].come_alive_p95_s == 0 for resident in target[1]), + -self.in_use(*target), + target[0].name, + ), + ) + + def packings( + self, + identity: str, + target: tuple[PlanLane, frozenset[str], frozenset[str]], + layout: _Layout, + choices: Mapping[str, _Choice], + ) -> list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]]: + lane, pinned, lazy = target + level = self.levels[identity] + order, receipt = min( + (pair for pair in self.receipts.get(identity, []) if _runs_on(pair[1], lane)), + key=lambda pair: (pair[1].throughput.requests_per_second, pair[0]), + ) + owner = layout.owners.get(lane.name) + host = None if owner is not None else lane.name + offered = choices[identity] + hosts = {option.host for option in offered.options if option.warmth is Warmth.LAZY_RESIDENT} + hosts.update(request.lane for request in offered.boot_requests if request.warmth is Warmth.LAZY_RESIDENT) + packings: list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]] = [] + current_class = self.current_class(identity) + blocked = (*self.warm_blockers(level, receipt, current_class), *self.reliability_blockers(level, lane.cells)) + load = self.boots.get((identity, lane.cell, lane.container_shape, Warmth.LAZY_RESIDENT)) + if ( + level.come_alive_p95_s != 0 + and not blocked + and lane.name not in hosts + and (load is None or load.seconds is not None) + and (load is None or load.seconds is None or _come_alive_blocker(level, load.seconds) is None) + ): + memory = self.memory(lane, pinned=pinned, lazy=lazy | {identity}) + if memory.fits is not False: + missing = ( + *(() if load is not None else (Measurement.BOOT_EVIDENCE,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + option = _Option( + receipt=receipt, + order=order, + warmth=Warmth.LAZY_RESIDENT, + cost=self.riding_cost(lane, identity, receipt), + come_alive_s=0.0 if load is None or load.seconds is None else load.seconds, + basis=ComeAliveBasis.BOOT_EVIDENCE if load is None else load.basis, + blockers=(), + current=False, + boot_s=None if load is None else load.boot_s, + host=host, + host_owner=owner, + ) + packings.append(({identity: option}, missing, ())) + members = pinned | {identity} + if all(choices[member].chosen is not None for member in pinned): + together = {member: self.co_receipt(member, lane, members - {member}) for member in members} + measured = {member: pair for member, pair in together.items() if pair is not None} + if not any( + ( + *self.warm_blockers(self.levels[member], pair[1], self.current_class(member)), + *self.reliability_blockers(self.levels[member], lane.cells), + ) + for member, pair in measured.items() + ): + memory = self.memory(lane, pinned=members, lazy=lazy) + if memory.fits is not False: + missing = ( + *(() if len(measured) == len(members) else (Measurement.CO_RESIDENCY_RECEIPT,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + own_order, own = measured.get(identity, (order, receipt)) + updates = { + identity: _Option( + receipt=own, + order=own_order, + warmth=Warmth.PINNED, + cost=self.riding_cost(lane, identity, own), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=(), + current=False, + host=host, + host_owner=owner, + ) + } + if not missing: + for member in sorted(pinned): + chosen = choices[member].chosen + member_order, member_receipt = measured[member] + if chosen is not None: + rate = member_receipt.throughput.requests_per_second + updates[member] = replace( + chosen, + receipt=member_receipt, + order=member_order, + cost=self.hourly(member_receipt) + * self.window.warm_container_hours(choices[member].requests, rate), + ) + packings.append((updates, missing, tuple(sorted(pinned)))) + return packings + + def pack(self, choices: dict[str, _Choice]) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: + layout = self.assemble(choices) + total = self.total(layout) + lanes = {lane.name: lane for lane in layout.plan.lanes} + hosted = Counter(entry.lane for entry in layout.plan.identities) + hours = self.parameters.hours_per_month + + def floor_cost(identity: str) -> float: + lane = lanes[layout.names[identity]] + if _lane_refusal(lane, self.prices) is not None: + return 0.0 + return round(self.lane_hourly(lane) * lane.envelope.floor_min * hours, 6) + + candidates = [ + identity + for identity, choice in choices.items() + if (chosen := choice.chosen) is not None + and chosen.host is None + and chosen.host_owner is None + and chosen.warmth is not Warmth.LAZY_RESIDENT + and identity not in self.generation + and hosted[layout.names[identity]] == 1 + ] + requests: dict[tuple[str, str, Warmth], MeasurementRequest] = {} + for identity in sorted(candidates, key=lambda identity: (-floor_cost(identity), identity)): + if hosted.get(layout.names.get(identity, ""), 0) != 1: + continue + moved = False + for target in self.pack_targets(identity, layout): + for updates, missing, beside in self.packings(identity, target, layout, choices): + members = frozenset(beside) | {identity} + trial = { + **choices, + **{ + member: replace(choices[member], chosen=option) + if member == identity + else replace(choices[member], chosen=option, beside=members - {member}) + for member, option in updates.items() + }, + } + trial_layout = self.assemble(trial) + trial_total = self.total(trial_layout) + saving = total - trial_total + if round(saving, 2) <= 0: + continue + option = updates[identity] + if missing: + key = (identity, target[0].name, option.warmth) + known = requests.get(key) + if known is None or saving > (known.saving_bound or 0.0): + requests[key] = MeasurementRequest( + identity=identity, + cell=target[0].cell, + container_shape=target[0].container_shape, + concurrency=target[0].concurrency, + lane=target[0].name, + measure=missing, + warmth=option.warmth, + reason=RequestReason.UNMEASURED, + cost_bound=option.cost, + saving_bound=saving, + co_residents=beside, + ) + continue + choices, layout, total = trial, trial_layout, trial_total + hosted = Counter(entry.lane for entry in layout.plan.identities) + moved = True + break + if moved: + break + return choices, list(requests.values()) + + def stale_beside(self, choices: Mapping[str, _Choice]) -> dict[str, frozenset[str]]: + layout = self.assemble(choices) + pinned: defaultdict[str, set[str]] = defaultdict(set) + for entry in layout.plan.identities: + if entry.warmth is Warmth.PINNED: + pinned[entry.lane].add(entry.identity) + stale: dict[str, frozenset[str]] = {} + for identity, choice in choices.items(): + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + continue + if self.stays(identity, choice, layout): + beside = frozenset(pinned[current[1].name] - {identity}) + if beside != choice.beside: + stale[identity] = beside + return stale + + def settle(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: + for _ in range(len(self.levels) + 1): + stale = self.stale_beside(choices) + if not stale: + return choices + choices = {**choices, **{identity: self.choose(identity, beside) for identity, beside in stale.items()}} + return choices + + def decide(self) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: + never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] + self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) + originals = {identity: self.choose(identity) for identity in sorted(self.levels)} + decided = self.stay_beside(self.settle(dict(originals)), originals) + decided, requests = self.pack(decided) + return self.settle(decided), requests + + def run(self, *, price_savings: bool = True) -> PlannerResult: + decided, packing_requests = self.decide() + choices = [decided[identity] for identity in sorted(decided)] + layout = self.assemble(decided) + new_plan, names, kept, receipts = layout.plan, layout.names, layout.kept, layout.receipts lane_reports, costs = self._pricing(new_plan, receipts) total = sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) reports: list[IdentityReport] = [] - requests: list[MeasurementRequest] = [] + requests: list[MeasurementRequest] = list(packing_requests) levers: list[Lever] = [] scarcity: list[Scarcity] = [] for choice in choices: @@ -1374,12 +1885,15 @@ def _identity_report( ) return replace(report, unplaced_reason=reason) rate = chosen.pricing_receipt.throughput.requests_per_second + summary = chosen.summary() + if chosen.host_owner is not None: + summary = replace(summary, lane=names[choice.identity]) return replace( report, status=IdentityStatus.PLACED, lane=names[choice.identity], monthly_cost=cost, - chosen=chosen.summary(), + chosen=summary, peak_containers=max((_Window.needed(count, rate) for count in choice.requests), default=0), next_cheaper=next_cheaper, ) @@ -1388,6 +1902,7 @@ def _pricing( self, plan: Plan, receipts: Mapping[str, Receipt | None] ) -> tuple[tuple[LaneReport, ...], dict[str, float]]: residents: defaultdict[str, list[str]] = defaultdict(list) + warmths = {entry.identity: entry.warmth for entry in plan.identities} for entry in plan.identities: residents[entry.lane].append(entry.identity) hours = self.parameters.hours_per_month @@ -1404,6 +1919,13 @@ def _pricing( demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} costs.update((identity, hourly * (floor + value)) for identity, value in priced.items()) + memory = _UNPACKABLE + if len(hosted) > 1: + memory = self.memory( + lane, + pinned=(identity for identity in hosted if warmths[identity] is Warmth.PINNED), + lazy=(identity for identity in hosted if warmths[identity] is Warmth.LAZY_RESIDENT), + ) reports.append( LaneReport( name=lane.name, @@ -1413,6 +1935,8 @@ def _pricing( floor_monthly_cost=hourly * lane.envelope.floor_min * hours, envelope_max_monthly_cost=hourly * lane.envelope.floor_max * hours, unpriced_identities=tuple(identity for identity in hosted if identity not in priced), + memory_mib=memory.need, + memory_limit_mib=memory.limit if memory.need is not None else None, ) ) return tuple(reports), costs @@ -1429,6 +1953,16 @@ def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | Non return ScarcityReason.PLACEMENT_WAIT +def _stands_beside(receipt: Receipt, beside: frozenset[str]) -> bool: + return beside <= frozenset(receipt.co_residents) if receipt.co_residents else not beside + + +def _p99(receipt: Receipt) -> float: + return max( + percentiles.p99_ms for percentiles in (receipt.latency, receipt.ttft, receipt.itl) if percentiles is not None + ) + + def _saving(total: float, replanned: float) -> float: saving = total - replanned return saving if round(saving, 2) > 0 else 0.0 diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index d63ffc25d..0f0935ce2 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -45,6 +45,8 @@ class ComeAliveBasis(StrEnum): class Measurement(StrEnum): RECEIPT = "receipt" BOOT_EVIDENCE = "boot-evidence" + FOOTPRINT = "footprint" + CO_RESIDENCY_RECEIPT = "co-residency-receipt" class RequestReason(StrEnum): @@ -108,6 +110,7 @@ class Option: lane: str | None = None fallback_cells: tuple[Cell, ...] = () placement_wait_s: float | None = None + co_residents: tuple[str, ...] = () def to_dict(self) -> dict[str, object]: return _compact( @@ -117,6 +120,7 @@ def to_dict(self) -> dict[str, object]: "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, + "coResidents": list(self.co_residents) or None, "warmth": self.warmth.value, "monthlyCost": _money(self.monthly_cost), "saving": None if self.saving is None else _money(self.saving), @@ -173,6 +177,8 @@ class LaneReport: envelope_max_monthly_cost: float | None = None unpriced_reason: str | None = None unpriced_identities: tuple[str, ...] = () + memory_mib: int | None = None + memory_limit_mib: float | None = None @property def unpriced(self) -> bool: @@ -191,6 +197,8 @@ def to_dict(self) -> dict[str, object]: ), "unpricedReason": self.unpriced_reason, "unpricedIdentities": list(self.unpriced_identities) or None, + "memoryMiB": self.memory_mib, + "memoryLimitMiB": self.memory_limit_mib, } ) @@ -223,6 +231,7 @@ class MeasurementRequest: container_shape: ContainerShape | None = None concurrency: int | None = None lane: str | None = None + co_residents: tuple[str, ...] = () def to_dict(self) -> dict[str, object]: return _compact( @@ -232,6 +241,7 @@ def to_dict(self) -> dict[str, object]: "containerShape": None if self.container_shape is None else self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, + "coResidents": list(self.co_residents) or None, "measure": [measurement.value for measurement in self.measure], "warmth": self.warmth.value, "reason": self.reason.value, diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index 93b9d9cd0..9248f33d6 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -137,6 +137,12 @@ def _string(self, key: str, value: object) -> str: raise SchemaError(self._at(key), f"expected a string, got {_describe(value)}") return value + @staticmethod + def text(value: object, path: str) -> str: + if not isinstance(value, str): + raise SchemaError(path, f"expected a string, got {_describe(value)}") + return value + def _boolean(self, key: str, value: object) -> bool: if not isinstance(value, bool): raise SchemaError(self._at(key), f"expected a boolean, got {_describe(value)}") @@ -605,10 +611,21 @@ class Receipt: latency: Percentiles | None = None ttft: Percentiles | None = None itl: Percentiles | None = None + footprint_mib: int | None = None + working_memory_mib: int | None = None + co_residents: tuple[str, ...] = () def __post_init__(self) -> None: _require_identifier("identity", self.identity) _require_at_least("concurrency", self.concurrency, 1) + _require_at_least("footprintMiB", self.footprint_mib, 1) + _require_at_least("workingMemoryMiB", self.working_memory_mib, 0) + for index, resident in enumerate(self.co_residents): + _require_identifier(f"coResidents[{index}]", resident) + if resident == self.identity: + raise SchemaError(f"coResidents[{index}]", f"names the receipt's own identity {resident!r}") + if resident in self.co_residents[:index]: + raise SchemaError(f"coResidents[{index}]", f"duplicates {resident!r}") if self.request_shape.operation is Operation.GENERATE: _require_absent("latency", self.latency, "for the generate operation, which reports ttft and itl") if self.ttft is None: @@ -641,6 +658,9 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: latency=reader.optional_child("latency", Percentiles.from_dict), ttft=reader.optional_child("ttft", Percentiles.from_dict), itl=reader.optional_child("itl", Percentiles.from_dict), + footprint_mib=reader.optional_integer("footprintMiB"), + working_memory_mib=reader.optional_integer("workingMemoryMiB"), + co_residents=reader.children("coResidents", _Reader.text, required=False), source=reader.child("source", SourceRef.from_dict), ) @@ -653,11 +673,14 @@ def to_dict(self) -> dict[str, object]: "cell": self.cell.to_dict(), "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, + "coResidents": list(self.co_residents) or None, "requestShape": self.request_shape.to_dict(), "throughput": self.throughput.to_dict(), "latency": None if self.latency is None else self.latency.to_dict(), "ttft": None if self.ttft is None else self.ttft.to_dict(), "itl": None if self.itl is None else self.itl.to_dict(), + "footprintMiB": self.footprint_mib, + "workingMemoryMiB": self.working_memory_mib, "source": self.source.to_dict(), } ) @@ -673,6 +696,7 @@ class BootEvidence: source: SourceRef boot_seconds: float | None = None placement_wait_s: float | None = None + footprint_mib: int | None = None borrowed: bool = False def __post_init__(self) -> None: @@ -692,6 +716,9 @@ def __post_init__(self) -> None: raise SchemaError("bootSeconds", "is required for an attempt that was placed") _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) + _require_at_least("footprintMiB", self.footprint_mib, 1) + if self.outcome is not BootOutcome.SUCCEEDED: + _require_absent("footprintMiB", self.footprint_mib, "for an attempt that did not succeed") @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -706,6 +733,7 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: placement_wait_s=reader.optional_number("placementWaitS"), boot_seconds=reader.optional_number("bootSeconds"), outcome=reader.enum("outcome", BootOutcome), + footprint_mib=reader.optional_integer("footprintMiB"), borrowed=reader.optional_boolean("borrowed") is True, source=reader.child("source", SourceRef.from_dict), ) @@ -722,6 +750,7 @@ def to_dict(self) -> dict[str, object]: "placementWaitS": self.placement_wait_s, "bootSeconds": self.boot_seconds, "outcome": self.outcome.value, + "footprintMiB": self.footprint_mib, "borrowed": True if self.borrowed else None, "source": self.source.to_dict(), } @@ -735,6 +764,7 @@ class MachineProfilePrice: accelerator_per_hour: float core_per_hour: float memory_gib_per_hour: float + gpu_memory_mib: int | None = None def __post_init__(self) -> None: _require_identifier("name", self.name) @@ -742,6 +772,7 @@ def __post_init__(self) -> None: _require_at_least("acceleratorPerHour", self.accelerator_per_hour, 0) _require_at_least("corePerHour", self.core_per_hour, 0) _require_at_least("memoryGiBPerHour", self.memory_gib_per_hour, 0) + _require_at_least("gpuMemoryMiB", self.gpu_memory_mib, 1) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -750,19 +781,23 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: cls, name=reader.string("name"), gpu_product=reader.string("gpuProduct"), + gpu_memory_mib=reader.optional_integer("gpuMemoryMiB"), accelerator_per_hour=reader.number("acceleratorPerHour"), core_per_hour=reader.number("corePerHour"), memory_gib_per_hour=reader.number("memoryGiBPerHour"), ) def to_dict(self) -> dict[str, object]: - return { - "name": self.name, - "gpuProduct": self.gpu_product, - "acceleratorPerHour": self.accelerator_per_hour, - "corePerHour": self.core_per_hour, - "memoryGiBPerHour": self.memory_gib_per_hour, - } + return _compact( + { + "name": self.name, + "gpuProduct": self.gpu_product, + "gpuMemoryMiB": self.gpu_memory_mib, + "acceleratorPerHour": self.accelerator_per_hour, + "corePerHour": self.core_per_hour, + "memoryGiBPerHour": self.memory_gib_per_hour, + } + ) @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index de243df5b..91a4f0861 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -47,13 +47,15 @@ "memoryMiB": 16384 }, "concurrency": 16, + "coResidents": [ + "intfloat/e5-base-v2" + ], "warmth": "pinned", "monthlyCost": 759.2, "comeAliveS": 0.0, "comeAliveBasis": "warm-floor", "receipt": { - "uri": "receipts/baai-bge-m3/l4-c16-16384.json", - "sha256": "0000000000000000000000000000000000000000000000000000000000000001" + "uri": "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" } }, "peakContainers": 1, @@ -407,8 +409,11 @@ }, "concurrency": 16, "lane": "default-l4-default", + "coResidents": [ + "BAAI/bge-m3" + ], "measure": [ - "receipt" + "co-residency-receipt" ], "warmth": "pinned", "reason": "unmeasured", @@ -477,7 +482,8 @@ "concurrency": 16, "lane": "default-l4-default", "measure": [ - "boot-evidence" + "boot-evidence", + "footprint" ], "warmth": "lazy-resident", "reason": "unmeasured", diff --git a/packages/sie_placement/tests/fixtures/planner/price-list.json b/packages/sie_placement/tests/fixtures/planner/price-list.json index 08d835d8a..871410550 100644 --- a/packages/sie_placement/tests/fixtures/planner/price-list.json +++ b/packages/sie_placement/tests/fixtures/planner/price-list.json @@ -6,6 +6,7 @@ { "name": "l4", "gpuProduct": "NVIDIA-L4", + "gpuMemoryMiB": 23034, "acceleratorPerHour": 0.8, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -13,6 +14,7 @@ { "name": "l40s", "gpuProduct": "NVIDIA-L40S", + "gpuMemoryMiB": 46068, "acceleratorPerHour": 1.2, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -20,6 +22,7 @@ { "name": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB", + "gpuMemoryMiB": 81920, "acceleratorPerHour": 2.5, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -27,6 +30,7 @@ { "name": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3", + "gpuMemoryMiB": 81559, "acceleratorPerHour": 4.0, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 diff --git a/packages/sie_placement/tests/fixtures/planner/receipts.json b/packages/sie_placement/tests/fixtures/planner/receipts.json index c29fe2621..31c080c66 100644 --- a/packages/sie_placement/tests/fixtures/planner/receipts.json +++ b/packages/sie_placement/tests/fixtures/planner/receipts.json @@ -26,11 +26,49 @@ "p95Ms": 171.9, "p99Ms": 214 }, + "footprintMiB": 2310, + "workingMemoryMiB": 1480, "source": { "uri": "receipts/baai-bge-m3/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000001" } }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "coResidents": [ + "intfloat/e5-base-v2" + ], + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 36 + }, + "latency": { + "p50Ms": 110.2, + "p95Ms": 171.9, + "p99Ms": 236 + }, + "footprintMiB": 2310, + "workingMemoryMiB": 1720, + "source": { + "uri": "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" + } + }, { "kind": "placement-receipt", "schemaVersion": 1, @@ -89,6 +127,8 @@ "p50Ms": 101, "p99Ms": 190 }, + "footprintMiB": 2290, + "workingMemoryMiB": 1210, "source": { "uri": "receipts/baai-bge-m3-sparse/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000003" @@ -194,6 +234,8 @@ "latency": { "p99Ms": 180 }, + "footprintMiB": 1650, + "workingMemoryMiB": 980, "source": { "uri": "receipts/qwen-qwen3-reranker-0.6b/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000006" @@ -225,6 +267,8 @@ "p50Ms": 140, "p99Ms": 280 }, + "footprintMiB": 8420, + "workingMemoryMiB": 2650, "source": { "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000007" diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index eac0a3852..7dca28e73 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -55,6 +55,7 @@ A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") H100 = Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3") RATES = {L4: 1.0, A100: 3.0, H100: 4.0} +CARD_MIB = {L4: 23034, A100: 81920, H100: 81559} SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) IDENTITY = "org/model" @@ -68,6 +69,7 @@ accelerator_per_hour=rate, core_per_hour=0.0, memory_gib_per_hour=0.0, + gpu_memory_mib=CARD_MIB[cell], ) for cell, rate in RATES.items() ), @@ -76,7 +78,9 @@ UNRELIABLE = 1 / 3 -def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0) -> Receipt: +def receipt( + identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0, footprint=None, working=None +) -> Receipt: return Receipt( identity=identity, cell=cell, @@ -85,6 +89,8 @@ def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), + footprint_mib=footprint, + working_memory_mib=working, source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{rps}/{p99}"), ) @@ -302,7 +308,8 @@ def test_a_lazy_resident_load_waits_for_no_placement(): ) service_levels = levels(**{OTHER: 0, IDENTITY: 60}) boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), attempt(OTHER, boot=1.0, wait=500.0, index=1)] - result = run(service_levels, [receipt(OTHER), receipt()], boots, current=current) + receipts = [receipt(OTHER, footprint=4000, working=1000), receipt(footprint=2000, working=1000)] + result = run(service_levels, receipts, boots, current=current) chosen = result.report.identity(IDENTITY).chosen assert (chosen.warmth, chosen.lane, chosen.come_alive_s, chosen.placement_wait_s) == ( Warmth.LAZY_RESIDENT, diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index feed8adf7..aec54c2bf 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -59,7 +59,7 @@ L4_HOURLY = 1.0 + 4 * 0.1 -def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10.0): +def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10.0, beside=()): return Receipt( identity=identity, cell=cell, @@ -68,7 +68,10 @@ def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10. request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), - source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}"), + co_residents=tuple(beside), + source=SourceRef( + uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}/{'+'.join(beside)}" + ), ) @@ -265,15 +268,22 @@ def test_an_identity_staying_beside_a_neighbour_offers_no_lever_that_loosening_i bound, seconds, permitted ): bounds = {"org/a": bound, "org/b": 0} - receipts = [receipt("org/a"), receipt("org/b")] + receipts = [ + receipt("org/a"), + receipt("org/b"), + receipt("org/a", beside=["org/b"]), + receipt("org/b", beside=["org/a"]), + ] boots = [boot("org/a", seconds)] result = planned(bounds, receipts, boots, current=shared_lane("org/a", "org/b")) assert result.plan == shared_lane("org/a", "org/b") assert result.report.identity("org/a").chosen.monthly_cost == pytest.approx(L4_HOURLY * MONTH) + assert result.report.identity("org/a").chosen.co_residents == ("org/b",) assert result.report.come_alive_levers == () loosened = planned({**bounds, "org/a": permitted}, receipts, boots, current=result.plan) - assert loosened.report.identity("org/a").chosen.warmth is Warmth.COLD_WITH_BOUND - assert loosened.report.monthly_cost - result.report.monthly_cost == pytest.approx(cold_cost()) + assert loosened.plan == result.plan + assert loosened.report.identity("org/a").chosen.warmth is Warmth.PINNED + assert loosened.report.monthly_cost == pytest.approx(result.report.monthly_cost) @pytest.mark.parametrize(("bound", "seconds", "permitted"), [(0, 95.2, 96), (60, 240.0, 240)]) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py new file mode 100644 index 000000000..2449df044 --- /dev/null +++ b/packages/sie_placement/tests/test_packing.py @@ -0,0 +1,694 @@ +from __future__ import annotations + +import random +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import pytest +from sie_placement.planner import PlannerParameters, plan +from sie_placement.report import IdentityStatus, Measurement, RequestReason, UnplacedReason, render +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + RequestShape, + ServiceLevel, + ServiceLevels, + SourceRef, + Throughput, + Warmth, +) + +START = datetime(2026, 11, 2, tzinfo=UTC) +MONTH = 730.0 +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +L40S = Cell(machine_profile="l40s", gpu_product="NVIDIA-L40S") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +RATES = {L4: 1.0, L40S: 2.0, A100: 3.0} +CARD_MIB = {L4: 23034, L40S: 46068, A100: 81920} +LIMIT = 23034 * 0.8 +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) +HOST = "org/host" +GUEST = "org/guest" + + +def prices(*, cards=True) -> PriceList: + return PriceList( + currency="USD", + machine_profiles=tuple( + MachineProfilePrice( + name=cell.machine_profile, + gpu_product=cell.gpu_product, + accelerator_per_hour=rate, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + gpu_memory_mib=CARD_MIB[cell] if cards else None, + ) + for cell, rate in RATES.items() + ), + placements=(Placement(name="test", region_multiplier=1),), + ) + + +def receipt( + identity, + cell=L4, + *, + beside=(), + footprint=None, + working=None, + p99=100.0, + rps=10.0, + shape=SHAPE, + concurrency=8, +) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=rps), + latency=Percentiles(p99_ms=p99), + co_residents=tuple(beside), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef( + uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{'+'.join(beside)}/{p99}" + ), + ) + + +def generation_receipt(identity, cell=L4, *, footprint=None, working=None) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=SHAPE, + concurrency=8, + request_shape=RequestShape(operation=Operation.GENERATE, items_per_request=1, output_tokens=128), + throughput=Throughput(requests_per_second=2.0, output_tokens_per_second=256.0), + ttft=Percentiles(p99_ms=500.0), + itl=Percentiles(p99_ms=40.0), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/generate"), + ) + + +def load(identity, cell=L4, seconds=20.0, *, footprint=None, outcome=BootOutcome.SUCCEEDED) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=SHAPE, + warmth=Warmth.LAZY_RESIDENT, + outcome=outcome, + boot_seconds=seconds, + footprint_mib=footprint, + source=SourceRef(uri=f"loads/{identity}/{cell.machine_profile}"), + ) + + +def cold(identity, cell=L4, seconds=30.0) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=SHAPE, + warmth=Warmth.COLD_WITH_BOUND, + outcome=BootOutcome.SUCCEEDED, + boot_seconds=seconds, + source=SourceRef(uri=f"boots/{identity}/{cell.machine_profile}"), + ) + + +def levels(**bounds) -> ServiceLevels: + return ServiceLevels( + identities=tuple( + IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + for identity, bound in sorted(bounds.items()) + ) + ) + + +def generation_level(identity, bound) -> IdentityServiceLevel: + return IdentityServiceLevel( + identity=identity, level=ServiceLevel(come_alive_p95_s=bound, ttft_p99_ms=1000, itl_p99_ms=80) + ) + + +def demand(hourly=None) -> Demand: + return Demand( + start=START, + end=START + timedelta(hours=24), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=count) + for hour, count in sorted(counts.items()) + ), + ) + for identity, counts in sorted((hourly or {}).items()) + ), + ) + + +def run(service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, **overrides): + parameters = PlannerParameters( + **({"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides) + ) + return plan(service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters) + + +def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: + return PlanLane( + name=name, + queue_pool="default", + bundle="default", + cell=cell, + container_shape=shape, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=floor, floor_max=floor), + ) + + +def plan_of(lanes, residents) -> Plan: + return Plan( + placement="test", + lanes=tuple(lanes), + identities=tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) + for identity, (name, warmth) in sorted(residents.items()) + ), + ) + + +def placements(result) -> dict[str, tuple[str, Warmth]]: + return {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} + + +def rendered(result) -> tuple[str, str]: + return render(result.plan.to_dict()), render(result.report.to_dict()) + + +def host_and_guest(guest_footprint=4000, guest_working=1500, *, guest_receipt=True): + receipts = [receipt(HOST, footprint=9000, working=2000)] + if guest_receipt: + receipts.append(receipt(GUEST, footprint=guest_footprint, working=guest_working)) + return receipts + + +def test_a_lazy_resident_joins_a_lane_formed_in_the_same_run_when_its_memory_fits(): + result = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)]) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert result.report.monthly_cost == pytest.approx(MONTH) + chosen = result.report.identity(GUEST).chosen + assert (chosen.lane, chosen.warmth, chosen.come_alive_s, chosen.monthly_cost) == ( + "org-host-l4", + Warmth.LAZY_RESIDENT, + 20.0, + 0, + ) + (planned,) = result.report.lanes + assert (planned.memory_mib, planned.memory_limit_mib) == (9000 + 4000 + 2000, pytest.approx(LIMIT)) + assert planned.to_dict()["memoryMiB"] == 15000 + again = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_lane_that_cannot_hold_the_resident_with_its_working_memory_hosts_none(): + result = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(guest_footprint=8000), [load(GUEST)]) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] == [] + + +def test_lazy_residents_add_their_largest_footprint_and_the_largest_working_memory(): + receipts = [ + receipt(HOST, footprint=9000, working=2000), + receipt("org/g1", footprint=4000, working=1000), + receipt("org/g2", footprint=3000, working=3500), + ] + boots = [load("org/g1"), load("org/g2")] + result = run(levels(**{HOST: 0, "org/g1": 60, "org/g2": 60}), receipts, boots) + assert placements(result) == { + HOST: ("org-host-l4", Warmth.PINNED), + "org/g1": ("org-host-l4", Warmth.LAZY_RESIDENT), + "org/g2": ("org-host-l4", Warmth.LAZY_RESIDENT), + } + (planned,) = result.report.lanes + assert planned.memory_mib == 9000 + 4000 + 3500 + receipts[2] = receipt("org/g2", footprint=3000, working=6000) + result = run(levels(**{HOST: 0, "org/g1": 60, "org/g2": 60}), receipts, boots) + assert placements(result)["org/g2"] == ("org-g2-l4", Warmth.PINNED) + assert {lane.name: lane.memory_mib for lane in result.report.lanes}["org-host-l4"] == 15000 + + +def test_a_packing_without_footprints_is_a_measurement_request_with_its_saving_bound(): + receipts = [receipt(HOST, footprint=9000, working=2000), receipt(GUEST)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)]) + assert placements(result)[GUEST] == ("org-guest-l4", Warmth.PINNED) + requests = [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] + assert [(request.identity, request.warmth, request.measure, request.co_residents) for request in requests] == [ + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT, Measurement.FOOTPRINT), (HOST,)), + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.FOOTPRINT,), ()), + ] + for request in requests: + assert (request.reason, request.cost_bound, request.concurrency) == (RequestReason.UNMEASURED, 0, 8) + assert request.saving_bound == pytest.approx(MONTH) + assert requests[1].to_dict()["measure"] == ["footprint"] + + +def test_a_packing_without_a_load_asks_for_the_load_and_the_footprints_it_lacks(): + receipts = [receipt(HOST), receipt(GUEST, footprint=4000, working=1500)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts) + lazy = [ + request + for request in result.report.measurement_requests + if request.lane == "org-host-l4" and request.warmth is Warmth.LAZY_RESIDENT + ] + assert [request.measure for request in lazy] == [(Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)] + + +@pytest.mark.parametrize( + ("price_list", "shape"), [(prices(cards=False), SHAPE), (prices(), PAIR)], ids=["no-card-memory", "two-cards"] +) +def test_a_lane_whose_card_memory_is_unknown_or_whose_container_holds_several_cards_packs_nothing(price_list, shape): + receipts = [ + receipt(HOST, footprint=9000, working=2000, shape=shape), + receipt(GUEST, footprint=4000, working=1500, shape=shape), + ] + boots = [replace(load(GUEST), container_shape=shape)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, boots, price_list=price_list) + assert placements(result)[GUEST] == ("org-guest-l4", Warmth.PINNED) + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + + +def pair_receipts(*, a_p99=180.0, b_p99=200.0): + return [ + receipt("org/a", footprint=6000, working=1000), + receipt("org/b", footprint=5000, working=1000), + receipt("org/a", beside=["org/b"], footprint=6000, working=1500, p99=a_p99, rps=8.0), + receipt("org/b", beside=["org/a"], footprint=5000, working=1500, p99=b_p99, rps=8.0), + ] + + +def test_two_never_cold_identities_share_a_card_on_their_co_residency_receipt(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts()) + assert placements(result) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert result.report.monthly_cost == pytest.approx(MONTH) + for identity, beside in (("org/a", ("org/b",)), ("org/b", ("org/a",))): + report = result.report.identity(identity) + assert (report.lane, report.chosen.co_residents) == ("org-b-l4", beside) + assert report.chosen.to_dict()["coResidents"] == list(beside) + assert result.report.identity("org/a").chosen.lane == "org-b-l4" + (planned,) = result.report.lanes + assert planned.memory_mib == 6000 + 5000 + 1500 + again = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.identity("org/a").chosen.co_residents == ("org/b",) + + +def test_a_co_residency_receipt_over_a_warm_bound_keeps_the_identities_apart(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(b_p99=300.0)) + assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert all(request.lane is None for request in result.report.measurement_requests) + + +def test_a_co_pinned_resident_is_judged_by_its_co_residency_receipt(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + kept = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current) + assert kept.plan == current + assert kept.report.identity("org/a").chosen.co_residents == ("org/b",) + split = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(a_p99=300.0), current=current) + assert placements(split) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)} + assert split.report.identity("org/b").chosen.co_residents == () + again = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(a_p99=300.0), current=split.plan) + assert again.report.unchanged + + +def test_a_current_co_pinned_lane_without_its_co_residency_receipt_is_kept_and_asks_for_it(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts()[:2], current=current) + assert result.plan == current + for identity, beside in (("org/a", ("org/b",)), ("org/b", ("org/a",))): + report = result.report.identity(identity) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + (request,) = [request for request in result.report.measurement_requests if request.identity == identity] + assert (request.measure, request.lane, request.co_residents, request.saving_bound) == ( + (Measurement.CO_RESIDENCY_RECEIPT,), + "shared", + beside, + None, + ) + + +def test_a_co_residency_receipt_beside_more_residents_stands_for_fewer(): + current = plan_of( + [lane("shared")], + {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED), "org/c": ("shared", Warmth.PINNED)}, + ) + receipts = [ + receipt("org/a", beside=["org/b", "org/c"], p99=150.0), + receipt("org/b", beside=["org/a", "org/c"], p99=150.0), + receipt("org/c", beside=["org/a", "org/b"], p99=300.0), + receipt("org/c"), + ] + service_levels = levels(**{"org/a": 0, "org/b": 0, "org/c": 600}) + result = run(service_levels, receipts, [cold("org/c")], current=current) + assert placements(result) == { + "org/a": ("shared", Warmth.PINNED), + "org/b": ("shared", Warmth.PINNED), + "org/c": ("org-c-l4", Warmth.COLD_WITH_BOUND), + } + assert result.report.identity("org/a").status is IdentityStatus.PLACED + assert result.report.identity("org/a").chosen.co_residents == ("org/b", "org/c") + again = run(service_levels, receipts, [cold("org/c")], current=result.plan) + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + + +def test_generation_identities_are_never_packed(): + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 0), + IdentityServiceLevel(identity=GUEST, level=ServiceLevel(come_alive_p95_s=60, warm_p99_ms=250)), + ) + ) + receipts = [ + generation_receipt("org/llm", footprint=9000, working=2000), + receipt(GUEST, footprint=2000, working=500), + ] + result = run(service_levels, receipts, [load(GUEST)]) + assert placements(result) == {"org/llm": ("org-llm-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 600), + IdentityServiceLevel(identity=HOST, level=ServiceLevel(come_alive_p95_s=0, warm_p99_ms=250)), + ) + ) + receipts = [generation_receipt("org/llm", footprint=2000, working=500), receipt(HOST, footprint=9000, working=2000)] + result = run(service_levels, receipts, [load("org/llm")]) + assert placements(result)["org/llm"] == ("org-llm-l4", Warmth.PINNED) + + +def test_a_generation_identity_on_a_current_lane_hosts_no_lazy_resident(): + current = plan_of([lane("own"), lane("llm")], {"org/llm": ("llm", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 0), + IdentityServiceLevel(identity=GUEST, level=ServiceLevel(come_alive_p95_s=60, warm_p99_ms=250)), + ) + ) + receipts = [ + generation_receipt("org/llm", footprint=9000, working=2000), + receipt(GUEST, footprint=2000, working=500), + ] + result = run(service_levels, receipts, [load(GUEST)], current=current) + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + assert all(request.lane != "llm" for request in result.report.measurement_requests) + + +def test_packing_fills_the_lane_with_the_most_footprint_in_use_first(): + receipts = [ + receipt("org/h1", footprint=9000, working=1000), + receipt("org/h2", footprint=4000, working=1000), + receipt(GUEST, footprint=2000, working=1000), + ] + result = run(levels(**{"org/h1": 0, "org/h2": 0, GUEST: 60}), receipts, [load(GUEST)]) + assert placements(result) == { + "org/h1": ("org-h1-l4", Warmth.PINNED), + "org/h2": ("org-h2-l4", Warmth.PINNED), + GUEST: ("org-h1-l4", Warmth.LAZY_RESIDENT), + } + requests = [(request.identity, request.lane, request.warmth) for request in result.report.measurement_requests] + assert ("org/h2", "org-h1-l4", Warmth.PINNED) in requests + + +def test_packing_tries_lanes_a_never_cold_identity_pins_before_the_others(): + receipts = [ + receipt("org/a-warm", footprint=9000, working=1000), + receipt("org/b-never-cold", footprint=4000, working=1000), + receipt(GUEST, footprint=2000, working=1000), + ] + service_levels = levels(**{"org/a-warm": 60, "org/b-never-cold": 0, GUEST: 60}) + result = run(service_levels, receipts, [load(GUEST)]) + assert placements(result)[GUEST] == ("org-b-never-cold-l4", Warmth.LAZY_RESIDENT) + + +def test_a_host_that_can_itself_ride_a_never_cold_lane_takes_its_guests_along_in_one_run(): + receipts = [ + receipt("org/m1", footprint=5000, working=2500, rps=1.0), + receipt("org/m2", footprint=5000, working=500), + receipt("org/m3", footprint=9000, working=2500, rps=1.0), + ] + boots = [load("org/m1", seconds=45.0), load("org/m2", seconds=5.0), load("org/m3", seconds=45.0)] + service_levels = levels(**{"org/m1": 600, "org/m2": 0, "org/m3": 60}) + result = run(service_levels, receipts, boots) + assert placements(result) == { + "org/m1": ("org-m2-l4", Warmth.LAZY_RESIDENT), + "org/m2": ("org-m2-l4", Warmth.PINNED), + "org/m3": ("org-m2-l4", Warmth.LAZY_RESIDENT), + } + assert result.report.monthly_cost == pytest.approx(MONTH) + (planned,) = result.report.lanes + assert planned.memory_mib == 5000 + 9000 + 2500 + again = run(service_levels, receipts, boots, current=result.plan) + assert again.report.unchanged + + +@pytest.mark.parametrize(("footprint", "measure"), [(4000, None), (8000, ()), (None, (Measurement.FOOTPRINT,))]) +def test_the_lazy_resident_selection_on_a_current_lane_requires_the_memory_check(footprint, measure): + current = plan_of([lane("own"), lane("shared")], {HOST: ("shared", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + receipts = host_and_guest(guest_footprint=footprint) + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)], current=current) + requests = [ + request.measure + for request in result.report.measurement_requests + if request.lane == "shared" and request.warmth is Warmth.LAZY_RESIDENT + ] + if measure is None: + assert placements(result)[GUEST] == ("shared", Warmth.LAZY_RESIDENT) + assert requests == [] + else: + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + assert requests == ([measure] if measure else []) + + +def test_an_identity_whose_neighbours_departure_strips_its_co_residency_evidence_is_decided_again_in_the_run(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + receipts = [ + receipt("org/a", beside=["org/b"], p99=100.0), + receipt("org/b", beside=["org/a"], p99=300.0), + receipt("org/a", p99=300.0), + receipt("org/a", cell=A100), + receipt("org/b", cell=A100), + ] + service_levels = levels(**{"org/a": 0, "org/b": 0}) + result = run(service_levels, receipts, current=current) + assert placements(result) == { + "org/a": ("org-a-a100-80gb", Warmth.PINNED), + "org/b": ("org-b-a100-80gb", Warmth.PINNED), + } + again = run(service_levels, receipts, current=result.plan) + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + + +def test_packing_visits_identities_in_decreasing_floor_cost(): + current = plan_of( + [lane("host"), lane("double", floor=2)], + {HOST: ("host", Warmth.PINNED), "org/z": ("double", Warmth.PINNED)}, + ) + receipts = [ + receipt(HOST, footprint=9000, working=1000), + receipt("org/y", footprint=5000, working=1000), + receipt("org/z", footprint=5000, working=1000), + receipt(HOST, beside=["org/y"], footprint=9000, working=1000), + receipt("org/y", beside=[HOST], footprint=5000, working=1000), + receipt(HOST, beside=["org/z"], footprint=9000, working=1000), + receipt("org/z", beside=[HOST], footprint=5000, working=1000), + ] + result = run(levels(**{HOST: 0, "org/y": 0, "org/z": 0}), receipts, current=current) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + "org/y": ("org-y-l4", Warmth.PINNED), + "org/z": ("host", Warmth.PINNED), + } + assert result.report.monthly_cost == pytest.approx(2 * MONTH) + assert all(request.identity != "org/y" or request.lane is None for request in result.report.measurement_requests) + again = run(levels(**{HOST: 0, "org/y": 0, "org/z": 0}), receipts, current=result.plan) + assert again.report.unchanged + + +def test_a_pinned_identity_stays_beside_a_neighbour_the_lane_stays_for(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + receipts = [*pair_receipts(), receipt("org/b", cell=A100)] + result = run(levels(**{"org/a": 60, "org/b": 0}), receipts, [cold("org/a")], current=current) + assert result.plan == current + assert result.report.monthly_cost == pytest.approx(MONTH) + report = result.report.identity("org/a") + assert report.chosen.warmth is Warmth.PINNED + both = run(levels(**{"org/a": 60, "org/b": 60}), receipts, [cold("org/a"), cold("org/b")], current=current) + assert placements(both) == { + "org/a": ("org-a-l4", Warmth.COLD_WITH_BOUND), + "org/b": ("org-b-l4", Warmth.COLD_WITH_BOUND), + } + assert both.report.monthly_cost == 0 + + +def test_identities_that_leave_a_lane_together_return_when_sharing_it_costs_less(): + current = plan_of( + [lane("shared", cell=A100)], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)} + ) + receipts = [ + receipt("org/a", cell=A100, beside=["org/b"]), + receipt("org/b", cell=A100, beside=["org/a"]), + receipt("org/a", cell=L40S), + receipt("org/b", cell=L40S), + ] + result = run(levels(**{"org/a": 0, "org/b": 0}), receipts, current=current) + assert result.plan == current + assert result.report.monthly_cost == pytest.approx(3 * MONTH) + alone = plan_of([lane("shared", cell=A100)], {"org/a": ("shared", Warmth.PINNED)}) + single = run(levels(**{"org/a": 0}), receipts, current=alone) + assert placements(single) == {"org/a": ("org-a-l40s", Warmth.PINNED)} + + +IDENTITIES = ("org/m0", "org/m1", "org/m2", "org/m3", "org/m4") + + +@pytest.mark.parametrize("seed", range(150)) +def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = list(IDENTITIES[: rng.randint(2, 5)]) + generation = {identity for identity in identities if rng.random() < 0.15} + entries = [] + for identity in identities: + bound = rng.choice([0, 0, 30, 60, 600]) + if identity in generation: + entries.append(generation_level(identity, bound)) + else: + entries.append( + IdentityServiceLevel( + identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=rng.choice([150, 250])) + ) + ) + service_levels = ServiceLevels(identities=tuple(entries)) + footprints = {identity: rng.choice([None, 2000, 5000, 9000]) for identity in identities} + working = {identity: rng.choice([None, 500, 2500]) for identity in identities} + receipts = [] + for identity in identities: + for cell in (L4, A100): + if rng.random() < 0.8: + if identity in generation: + receipts.append(generation_receipt(identity, cell, footprint=footprints[identity], working=500)) + else: + receipts.append( + receipt( + identity, + cell, + footprint=footprints[identity], + working=working[identity], + p99=rng.choice([100.0, 200.0]), + rps=rng.choice([1.0, 10.0]), + ) + ) + plain = [identity for identity in identities if identity not in generation] + for _ in range(rng.randint(0, 4)): + group = rng.sample(plain, min(len(plain), rng.randint(2, 3))) if len(plain) >= 2 else [] + for identity in group: + receipts.append( + receipt( + identity, + beside=sorted(set(group) - {identity}), + footprint=footprints[identity], + working=rng.choice([500, 2500]), + p99=rng.choice([150.0, 300.0]), + rps=rng.choice([1.0, 10.0]), + ) + ) + boots = [load(identity, seconds=rng.choice([5.0, 45.0, 200.0])) for identity in identities if rng.random() < 0.7] + boots.extend(cold(identity, seconds=rng.choice([20.0, 300.0])) for identity in identities if rng.random() < 0.3) + hourly = { + identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} + for identity in identities + } + current = None + if rng.random() < 0.5: + first = run(service_levels, receipts, boots, hourly=hourly) + current = first.plan + result = run(service_levels, receipts, boots, hourly=hourly, current=current) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} + residents: dict[str, list[PlanIdentity]] = {} + for entry in result.plan.identities: + residents.setdefault(entry.lane, []).append(entry) + before = {} if current is None else {planned.name: planned for planned in current.lanes} + reports = {lane_report.name: lane_report for lane_report in result.report.lanes} + for planned in result.plan.lanes: + hosted = residents[planned.name] + if any(entry.identity in generation for entry in hosted): + assert len(hosted) == 1 + if len(hosted) < 2: + continue + pinned = {entry.identity for entry in hosted if entry.warmth is Warmth.PINNED} + formed = planned.name not in before or { + entry.identity for entry in current.identities if entry.lane == planned.name + } != {entry.identity for entry in hosted} + if formed: + assert reports[planned.name].memory_mib is not None + assert reports[planned.name].memory_mib <= reports[planned.name].memory_limit_mib + for entry in hosted: + report = result.report.identity(entry.identity) + if report.chosen is None: + continue + level = levels_by_identity[entry.identity] + if level.warm_p99_ms is not None: + assert report.chosen.blocked_by == () + if entry.warmth is Warmth.PINNED and len(pinned) > 1: + assert set(report.chosen.co_residents) >= pinned - {entry.identity} + if entry.warmth is Warmth.LAZY_RESIDENT: + assert level.come_alive_p95_s != 0 + assert level.come_alive_p95_s is None or report.chosen.come_alive_s <= level.come_alive_p95_s + for request in result.report.measurement_requests: + assert request.saving_bound is None or request.saving_bound > 0 + again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + shuffled = list(receipts) + rng.shuffle(shuffled) + reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current) + assert rendered(reordered) == rendered(result) + for entry in service_levels.identities: + bound = entry.level.come_alive_p95_s + looser = ServiceLevels( + identities=tuple( + replace(other, level=replace(other.level, come_alive_p95_s=bound + 600)) + if other.identity == entry.identity + else other + for other in service_levels.identities + ) + ) + loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan) + assert loosened.report.monthly_cost <= result.report.monthly_cost + 0.01 diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 2bcbd0718..c4bf3886c 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -79,18 +79,26 @@ ) -def profile(cell: Cell, accelerator: float, core: float = 0.0) -> MachineProfilePrice: +CARD_MIB = {L4: 23034, A100: 81920} + + +def profile(cell: Cell, accelerator: float, core: float = 0.0, memory: int | None = None) -> MachineProfilePrice: return MachineProfilePrice( name=cell.machine_profile, gpu_product=cell.gpu_product, accelerator_per_hour=accelerator, core_per_hour=core, memory_gib_per_hour=0.0, + gpu_memory_mib=memory, ) -def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=()) -> PriceList: - profiles = [profile(cell, rate, core) for cell, rate in ((L4, l4), (A100, a100)) if rate is not None] +def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=(), cards=True) -> PriceList: + profiles = [ + profile(cell, rate, core, CARD_MIB[cell] if cards else None) + for cell, rate in ((L4, l4), (A100, a100)) + if rate is not None + ] return PriceList( currency="USD", machine_profiles=(*profiles, *extra), @@ -101,7 +109,20 @@ def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=()) -> Price L4_ONLY = price_list(a100=None) -def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0, uri=None) -> Receipt: +def receipt( + identity=IDENTITY, + cell=L4, + *, + shape=SHAPE, + concurrency=8, + rps=10.0, + p99=100.0, + uri=None, + co_residents=(), + footprint=None, + working=None, +) -> Receipt: + beside = "".join(f"+{name}" for name in co_residents) return Receipt( identity=identity, cell=cell, @@ -110,7 +131,12 @@ def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), - source=SourceRef(uri=uri or f"receipts/{identity}/{cell.machine_profile}-{concurrency}-{shape.memory_mib}"), + co_residents=tuple(co_residents), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef( + uri=uri or f"receipts/{identity}/{cell.machine_profile}-{concurrency}-{shape.memory_mib}{beside}" + ), ) @@ -269,7 +295,7 @@ def resident_estate() -> Plan: ) -RESIDENT_RECEIPTS = [receipt(identity=ANCHOR), receipt()] +RESIDENT_RECEIPTS = [receipt(identity=ANCHOR, footprint=6000, working=2000), receipt(footprint=3000, working=1000)] LOADS = [boot(warmth=Warmth.LAZY_RESIDENT, seconds=20.0, uri="loads/example.json")] @@ -736,16 +762,17 @@ def test_the_next_cheaper_option_is_the_nearest_cheaper_one_and_names_its_bound( def test_a_next_cheaper_saving_is_what_loosening_its_bounds_saves_the_plan(others, saving): service_levels = levels(*((name, ServiceLevel(come_alive_p95_s=0)) for name in (IDENTITY, *others))) current = single_lane_plan(others=others) - result = run(service_levels, [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=current) + receipts = [receipt(), *([receipt(co_residents=others)] if others else [])] + result = run(service_levels, receipts, [boot(seconds=30.0)], prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) assert (report.lane, report.chosen.warmth) == ("shared", Warmth.PINNED) assert (report.next_cheaper.warmth, report.next_cheaper.monthly_cost) == (Warmth.COLD_WITH_BOUND, 0.0) assert report.chosen.monthly_cost - report.next_cheaper.monthly_cost == pytest.approx(MONTH) assert report.next_cheaper.saving == pytest.approx(saving) again = run( - loosened(service_levels, IDENTITY, 30), [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=result.plan + loosened(service_levels, IDENTITY, 30), receipts, [boot(seconds=30.0)], prices=L4_ONLY, current=result.plan ) - assert again.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + assert again.report.identity(IDENTITY).chosen.warmth is (Warmth.PINNED if others else Warmth.COLD_WITH_BOUND) assert result.report.monthly_cost - again.report.monthly_cost == pytest.approx(saving) @@ -1220,21 +1247,19 @@ def test_a_load_over_the_bound_is_a_come_alive_lever(): assert lever.saving_under_bound == pytest.approx(MONTH) -def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_from_the_next_run(): +def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_in_the_same_run(): current = plan_of([plan_lane("own")], {IDENTITY: ("own", Warmth.PINNED)}) first = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) assert [(entry.identity, entry.lane, entry.warmth) for entry in first.plan.identities] == [ - (ANCHOR, "org-anchor-l4", Warmth.PINNED), - (IDENTITY, "own", Warmth.PINNED), - ] - second = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=first.plan) - assert [(entry.identity, entry.lane, entry.warmth) for entry in second.plan.identities] == [ (ANCHOR, "org-anchor-l4", Warmth.PINNED), (IDENTITY, "org-anchor-l4", Warmth.LAZY_RESIDENT), ] - assert first.report.monthly_cost - second.report.monthly_cost == pytest.approx(MONTH) - third = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=second.plan) - assert third.report.unchanged + assert first.report.monthly_cost == pytest.approx(MONTH) + chosen = first.report.identity(IDENTITY).chosen + assert (chosen.lane, chosen.come_alive_s, chosen.monthly_cost) == ("org-anchor-l4", 20.0, 0) + second = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=first.plan) + assert second.plan == first.plan + assert second.report.unchanged def test_a_lazy_resident_never_joins_a_lane_its_never_cold_neighbour_leaves(): @@ -1250,7 +1275,11 @@ def test_a_lazy_resident_never_joins_a_lane_its_never_cold_neighbour_leaves(): (IDENTITY, "own", Warmth.PINNED), ] assert result.report.monthly_cost == pytest.approx(2 * MONTH) - assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + assert all(request.lane != "shared" for request in result.report.measurement_requests) + lazy = [request for request in result.report.measurement_requests if request.warmth is Warmth.LAZY_RESIDENT] + assert [(request.lane, request.measure) for request in lazy] == [ + ("org-anchor-l4", (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)) + ] again = run(resident_levels(), receipts, loads, current=result.plan) assert again.report.unchanged @@ -1263,7 +1292,7 @@ def test_a_lazy_resident_whose_lane_keeps_no_never_cold_neighbour_pays_the_lane_ ("org/warm", ServiceLevel(come_alive_p95_s=120, warm_p99_ms=ANY_P99_MS)), (IDENTITY, ServiceLevel(come_alive_p95_s=60, warm_p99_ms=ANY_P99_MS)), ) - receipts = [receipt(identity="org/warm"), receipt()] + receipts = [receipt(identity="org/warm", footprint=6000, working=2000), receipt(footprint=3000, working=1000)] boots = [boot(identity="org/warm"), *LOADS, boot(seconds=30.0)] hourly = {"org/warm": {0: 1}, IDENTITY: {0: 1}} result = run(service_levels, receipts, boots, prices=L4_ONLY, hourly=hourly, current=current) @@ -1280,7 +1309,8 @@ def test_a_lazy_resident_whose_lane_keeps_no_never_cold_neighbour_pays_the_lane_ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_than_the_lane_floor(): current = plan_of([plan_lane("shared")], {IDENTITY: ("shared", Warmth.LAZY_RESIDENT)}) - result = run(level(come_alive_p95_s=60), [receipt()], LOADS, prices=L4_ONLY, current=current) + receipts = [receipt(footprint=3000, working=1000)] + result = run(level(come_alive_p95_s=60), receipts, LOADS, prices=L4_ONLY, current=current) assert result.plan == current assert result.report.unchanged report = result.report.identity(IDENTITY) @@ -1293,37 +1323,55 @@ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_th assert result.report.monthly_cost == pytest.approx(MONTH) -@pytest.mark.parametrize( - ("shared", "anchor_bound"), - [ - (plan_lane("shared"), 30), - (plan_lane("shared", bundle="sglang"), 0), - (plan_lane("shared", queue_pool="batch"), 0), - ], -) -def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_lane_of_its_pool_and_bundle(shared, anchor_bound): +@pytest.mark.parametrize("shared", [plan_lane("shared", bundle="sglang"), plan_lane("shared", queue_pool="batch")]) +def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): current = plan_of([plan_lane("own"), shared], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}) - result = run(resident_levels(anchor_bound=anchor_bound), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities} == { + ANCHOR: "org-anchor-l4", + IDENTITY: "org-anchor-l4", + } + assert all(entry.lane != "shared" for entry in result.plan.identities) + assert all(request.lane != "shared" for request in result.report.measurement_requests) + + +def test_a_current_lane_hosts_lazy_residents_beside_a_never_cold_neighbour_and_a_plan_lane_beside_any_pinned_one(): + current = plan_of( + [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} + ) + result = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) - assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert (report.lane, report.chosen.warmth, report.chosen.lane) == ("shared", Warmth.LAZY_RESIDENT, "shared") + assert result.report.monthly_cost == pytest.approx(MONTH) assert all(lever.warmth is not Warmth.LAZY_RESIDENT for lever in result.report.come_alive_levers) - assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + again = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=result.plan) + assert again.report.unchanged @pytest.mark.parametrize( - ("bound", "expected"), [(60, [(Warmth.COLD_WITH_BOUND, None), (Warmth.LAZY_RESIDENT, "shared")]), (0, [])] + ("bound", "expected"), + [ + ( + 60, + [ + (Warmth.COLD_WITH_BOUND, None, (Measurement.BOOT_EVIDENCE,)), + (Warmth.LAZY_RESIDENT, "shared", (Measurement.BOOT_EVIDENCE,)), + (Warmth.PINNED, "shared", (Measurement.CO_RESIDENCY_RECEIPT,)), + ], + ), + (0, [(Warmth.PINNED, "shared", (Measurement.CO_RESIDENCY_RECEIPT,))]), + ], ) def test_a_warm_lane_without_a_load_row_asks_for_one_at_the_saving_it_unlocks(bound, expected): result = run(resident_levels(bound=bound), RESIDENT_RECEIPTS, prices=L4_ONLY, current=resident_estate()) requests = [request for request in result.report.measurement_requests if request.identity == IDENTITY] - assert [(request.warmth, request.lane) for request in requests] == expected + assert [(request.warmth, request.lane, request.measure) for request in requests] == expected for request in requests: - assert (request.measure, request.reason, request.cost_bound) == ( - (Measurement.BOOT_EVIDENCE,), - RequestReason.UNMEASURED, - 0, - ) + assert (request.reason, request.cost_bound) == (RequestReason.UNMEASURED, 0) assert request.saving_bound == pytest.approx(MONTH) + (copin,) = [request for request in requests if request.warmth is Warmth.PINNED] + assert copin.co_residents == (ANCHOR,) + assert copin.to_dict()["coResidents"] == [ANCHOR] def test_a_current_placement_on_a_shared_lane_stays_beside_its_kept_neighbour(): @@ -1595,15 +1643,17 @@ def test_the_estate_report_states_its_findings(): if request.lane is not None ] assert on_lanes == [ - ("intfloat/e5-base-v2", (Measurement.RECEIPT,), Warmth.PINNED, "default-l4-default", None), + ("intfloat/e5-base-v2", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED, "default-l4-default", None), ( "BAAI/bge-m3:sparse", - (Measurement.BOOT_EVIDENCE,), + (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT), Warmth.LAZY_RESIDENT, "default-l4-default", pytest.approx(sparse.chosen.monthly_cost), ), ] + assert dense.chosen.co_residents == ("intfloat/e5-base-v2",) + assert dense.chosen.receipt.uri == "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" assert [change.identity for change in report.identity_changes] == ["BAAI/bge-m3:sparse", "Qwen/Qwen3-Embedding-4B"] assert report.monthly_cost == pytest.approx(sum(lane.monthly_cost for lane in report.lanes)) shared = {lane.name: lane for lane in report.lanes}["default-l4-default"] @@ -1646,8 +1696,9 @@ def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever_and_no_savin for bound in (9, 95): service_levels = loosened(inputs["service_levels"], "BAAI/bge-m3", bound) again = plan(**{**inputs, "service_levels": service_levels, "current_plan": result.plan}) - assert again.report.identity("BAAI/bge-m3").chosen.warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) - assert again.report.monthly_cost > result.report.monthly_cost + assert again.report.identity("BAAI/bge-m3").chosen.warmth is Warmth.PINNED + assert again.plan == result.plan + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) def test_an_estate_next_cheaper_saving_is_what_planning_again_with_its_bounds_loosened_saves(): diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index 841c8eead..e63d22c61 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -452,6 +452,79 @@ def test_capacity_built_in_code_names_type_ceilings_by_gpu_product(): assert caught.value.path == "gpuProduct" +def test_receipts_record_the_memory_a_model_holds_and_its_co_residents(): + data = load("receipt-encode.json") + data["coResidents"] = ["intfloat/e5-base-v2", "BAAI/bge-m3:sparse"] + data["footprintMiB"] = 2310 + data["workingMemoryMiB"] = 0 + receipt = Receipt.from_dict(data) + assert (receipt.co_residents, receipt.footprint_mib, receipt.working_memory_mib) == ( + ("intfloat/e5-base-v2", "BAAI/bge-m3:sparse"), + 2310, + 0, + ) + assert receipt.to_dict() == data + assert list(receipt.to_dict())[:7] == [ + "kind", + "schemaVersion", + "identity", + "cell", + "containerShape", + "concurrency", + "coResidents", + ] + plain = Receipt.from_dict(load("receipt-encode.json")) + assert (plain.co_residents, plain.footprint_mib, plain.working_memory_mib) == ((), None, None) + assert "coResidents" not in plain.to_dict() + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("coResidents", ["BAAI/bge-m3"], "$.coResidents[0]: names the receipt's own identity 'BAAI/bge-m3'"), + ("coResidents", ["org/a", "org/a"], "$.coResidents[1]: duplicates 'org/a'"), + ("coResidents", ["org/a", ""], "$.coResidents[1]: must be a non-empty string without whitespace, got ''"), + ("coResidents", [7], "$.coResidents[0]: expected a string, got number 7"), + ("coResidents", "org/a", "$.coResidents: expected an array, got string 'org/a'"), + ("footprintMiB", 0, "$.footprintMiB: must be at least 1, got 0"), + ("footprintMiB", 1.5, "$.footprintMiB: expected an integer, got number 1.5"), + ("workingMemoryMiB", -1, "$.workingMemoryMiB: must be at least 0, got -1"), + ], +) +def test_receipt_memory_and_co_residents_are_validated(field, value, message): + data = load("receipt-encode.json") + data[field] = value + assert str(schema_error(Receipt.from_dict, data)) == message + + +def test_boot_evidence_records_the_footprint_of_a_load_that_succeeded(): + data = load("boot-evidence.json") + data["footprintMiB"] = 2290 + evidence = BootEvidence.from_dict(data) + assert evidence.footprint_mib == 2290 + assert evidence.to_dict() == data + data["footprintMiB"] = 0 + assert str(schema_error(BootEvidence.from_dict, data)) == "$.footprintMiB: must be at least 1, got 0" + data["footprintMiB"] = 2290 + data["outcome"] = "failed" + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.footprintMiB: must be omitted for an attempt that did not succeed" + ) + + +def test_machine_profiles_may_state_the_memory_of_their_card(): + data = load("price-list.json") + data["machineProfiles"][0]["gpuMemoryMiB"] = 23034 + prices = PriceList.from_dict(data) + assert prices.machine_profiles[0].gpu_memory_mib == 23034 + assert prices.machine_profiles[1].gpu_memory_mib is None + assert prices.to_dict() == data + data["machineProfiles"][0]["gpuMemoryMiB"] = 0 + assert str(schema_error(PriceList.from_dict, data)) == ( + "$.machineProfiles[0].gpuMemoryMiB: must be at least 1, got 0" + ) + + def test_price_lists_validate_currency_rates_and_names(): data = load("price-list.json") data["currency"] = "usd" From f6f890807809cde4745d51aae747b5592f45fe4d Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 18:28:23 +0100 Subject: [PATCH 14/40] fix(placement): rank every cold attempt and hold lanes to the ceilings at maxContainers - The placement wait ranks every cold attempt on a GPU product. A placed attempt without placementWaitS counts as one that waited none, so the 20-attempt threshold and the 95th percentile no longer fall on a subset made mostly of unplaced attempts, and agree with the failure rate, which already counted every attempt. A product none of whose attempts records a wait or was unplaced still adds no wait. - The total and per-type ceilings now hold every covered lane at its maxContainers as well as at its floorMax, which is what a deployment compiler enforcing those ceilings refuses. The report gives peakAccelerators for those scopes, and the refusal names the container ceilings that exceed. The shared quota still bounds the envelope maxima. --- packages/sie_placement/README.md | 36 +++--- .../src/sie_placement/planner.py | 35 ++++-- .../sie_placement/src/sie_placement/report.py | 19 ++- .../sie_placement/tests/test_availability.py | 112 ++++++++++++++---- 4 files changed, 143 insertions(+), 59 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 304094fcf..caf70a449 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -116,9 +116,13 @@ obtain. Each field is optional, and at least one is declared: | `types` | Per `gpuProduct`, the `accelerators` the lanes that may be placed on that product may hold together | | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | -A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane -with fallback cells counts against the ceiling of every product it names, and -once against `total`. +Every container holds the `gpuCount` accelerators of its lane's container +shape. `total` and each of `types` must hold every lane they cover at its +container ceiling, `maxContainers` containers, at once, and so at its envelope +maximum, `floorMax` containers, too. `sharedQuota` bounds the envelope maxima, +the most a tuner may hold warm, together with `heldElsewhere`. A lane with +fallback cells counts against the ceiling of every product it names, and once +against `total`. ## Envelope @@ -147,8 +151,8 @@ result = plan( print(render(result.plan.to_dict()), render(result.report.to_dict())) ``` -`capacity` is optional. A plan whose envelope maxima exceed a capacity ceiling -is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` +`capacity` is optional. A plan whose lanes exceed a capacity ceiling, as +[Capacity](#capacity) states, is not returned: `plan` raises `CapacityExceededError`, a `PlannerInputError` whose message names each exceeded ceiling and the lanes that hold it, whose `exceeded` lists those ceilings, and whose `result` carries the refused plan and its report. @@ -194,12 +198,14 @@ document: same percentile. The boot is taken over the identity's attempts at that cell, container shape and warmth that got a container. The placement wait is taken over every cold attempt on the cell's GPU product, whatever the - identity, that records `placementWaitS` or was `unplaced`; a product - without such an attempt adds no wait. Below 20 attempts the slowest stands - for the percentile; from 20, their 95th percentile by nearest rank does, so - one slow attempt stops deciding once enough exist. An attempt that did not - succeed, a failed or timed-out boot or an unplaced placement, ranks above - every success, so the warmth is withdrawn when the percentile falls on one. + identity. An attempt that got a container without recording + `placementWaitS` waited none, and a product none of whose cold attempts + records `placementWaitS` or was `unplaced` adds no wait. Below 20 attempts + the slowest stands for the percentile; from 20, their 95th percentile by + nearest rank does, so one slow attempt stops deciding once enough exist. An + attempt that did not succeed, a failed or timed-out boot or an unplaced + placement, ranks above every success, so the warmth is withdrawn when the + percentile falls on one. The environment's own attempts replace borrowed ones, per cell, container shape and warmth for the boot and per GPU product for the placement wait. A `comeAliveP95S` of `0` admits `pinned` only. @@ -292,8 +298,9 @@ the ceiling. A floor raised on a zero-floor lane realizes as at least `fallback_floor`, so a zero-floor lane's nonzero `floorMax` is at least `fallback_floor` capped by the ceiling. The envelope carries the floor range only. A lane carried from the current plan keeps its envelope, capped by the -ceiling. Given a `capacity` document, the envelope maxima of every lane must -fit each of its ceilings, or the plan is refused as described above. A +ceiling. Given a `capacity` document, the lanes must fit each of its ceilings, +at their container ceilings and envelope maxima as [Capacity](#capacity) +states, or the plan is refused as described above. A loosened service level whose plan the ceilings would refuse saves nothing in the report. @@ -336,7 +343,8 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, the `lanes` that hold them, and whether it is + maxima hold against it, for `total` and `type` the `peakAccelerators` its + lanes hold at `maxContainers`, the `lanes` that hold them, and whether it is `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index bf286ef09..48e5ff752 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -439,14 +439,13 @@ def _placement_index(records: Iterable[BootEvidence]) -> dict[str, _Placement]: own = [record for record in group if not record.borrowed] used = own or group samples = [ - None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) - for record in used - if record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None + None if record.outcome is BootOutcome.UNPLACED else float(record.placement_wait_s or 0.0) for record in used ] + measured = any(record.outcome is BootOutcome.UNPLACED or record.placement_wait_s is not None for record in used) unplaced = sum(record.outcome is BootOutcome.UNPLACED for record in used) index[product] = _Placement( - wait_s=_percentile(samples) if samples else 0.0, - measured=bool(samples), + wait_s=_percentile(samples) if measured else 0.0, + measured=measured, failure_rate=unplaced / len(used), borrowed=not own, ) @@ -1138,17 +1137,26 @@ def capacity_use(self, plan: Plan) -> tuple[CapacityUse, ...]: if capacity is None: return () - def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: - counted = [lane for lane in lanes if lane.envelope.floor_max] - accelerators = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in counted) - return accelerators, tuple(lane.name for lane in counted) + def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: + lanes = tuple(lanes) + envelope = sum(lane.envelope.floor_max * lane.container_shape.gpu_count for lane in lanes) + peak = sum(lane.max_containers * lane.container_shape.gpu_count for lane in lanes) + return envelope, peak, tuple(lane.name for lane in lanes) - total, names = held(plan.lanes) + total, peak, names = held(plan.lanes) uses = [] if capacity.total is not None: - uses.append(CapacityUse(scope=CapacityScope.TOTAL, ceiling=capacity.total, accelerators=total, lanes=names)) + uses.append( + CapacityUse( + scope=CapacityScope.TOTAL, + ceiling=capacity.total, + accelerators=total, + peak_accelerators=peak, + lanes=names, + ) + ) for ceiling in sorted(capacity.types, key=lambda entry: entry.gpu_product): - accelerators, lanes = held( + accelerators, peak_accelerators, lanes = held( lane for lane in plan.lanes if any(cell.gpu_product == ceiling.gpu_product for cell in lane.cells) ) uses.append( @@ -1157,6 +1165,7 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: gpu_product=ceiling.gpu_product, ceiling=ceiling.accelerators, accelerators=accelerators, + peak_accelerators=peak_accelerators, lanes=lanes, ) ) @@ -1168,7 +1177,7 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, tuple[str, ...]]: ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=names, + lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), ) ) return tuple(uses) diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index d63ffc25d..29bbb4a34 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -315,28 +315,26 @@ class CapacityUse: lanes: tuple[str, ...] gpu_product: str | None = None held_elsewhere: int | None = None + peak_accelerators: int | None = None @property def exceeded(self) -> bool: - return self.accelerators > self.ceiling + return max(self.accelerators, self.peak_accelerators or 0) > self.ceiling def describe(self) -> str: lanes = ", ".join(self.lanes) or "no lane" - if self.scope is CapacityScope.TYPE: - return ( - f"the envelope maxima of {lanes} hold {self.accelerators} {self.gpu_product} accelerators, " - f"above the type ceiling of {self.ceiling}" - ) if self.scope is CapacityScope.SHARED_QUOTA: elsewhere = self.held_elsewhere or 0 return ( f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" ) - return ( - f"the envelope maxima of {lanes} hold {self.accelerators} accelerators, " - f"above the total ceiling of {self.ceiling}" - ) + accelerators = "accelerators" if self.gpu_product is None else f"{self.gpu_product} accelerators" + held = f"the envelope maxima of {lanes} hold {self.accelerators}" + if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: + held = f"the container ceilings of {lanes} hold {self.peak_accelerators}" + scope = "type" if self.scope is CapacityScope.TYPE else "total" + return f"{held} {accelerators}, above the {scope} ceiling of {self.ceiling}" def to_dict(self) -> dict[str, object]: return _compact( @@ -345,6 +343,7 @@ def to_dict(self) -> dict[str, object]: "gpuProduct": self.gpu_product, "ceiling": self.ceiling, "accelerators": self.accelerators, + "peakAccelerators": self.peak_accelerators, "heldElsewhere": self.held_elsewhere, "lanes": list(self.lanes), "exceeded": self.exceeded, diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index eac0a3852..ae226c927 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -225,7 +225,7 @@ def test_placement_waits_pool_over_every_attempt_on_a_gpu_type(): assert entry.to_dict()["reason"] == "placement-wait" -@pytest.mark.parametrize(("fast", "come_alive"), [(18, None), (19, 60.0), (37, None), (38, 60.0)]) +@pytest.mark.parametrize(("fast", "come_alive"), [(17, None), (18, 60.0), (36, None), (37, 60.0)]) def test_the_placement_wait_takes_the_95th_percentile_from_twenty_attempts(fast, come_alive): boots = [attempt(boot=50.0)] boots.extend(attempt(OTHER, boot=1.0, wait=10.0, index=index) for index in range(fast)) @@ -261,6 +261,35 @@ def test_an_unplaced_attempt_ranks_above_every_placed_one(placed, selected): assert entry.placement_failure_rate == pytest.approx(1 / (placed + 2)) +@pytest.mark.parametrize(("placed", "selected"), [(17, False), (18, True), (100, True)]) +def test_a_placed_attempt_without_a_placement_wait_ranks_as_one_that_waited_none(placed, selected): + boots = [attempt(boot=50.0)] + boots.extend(attempt(OTHER, boot=1.0, wait=0.0 if index % 2 else None, index=index) for index in range(placed)) + boots.append(unplaced(index=999)) + result = run(one(120), [receipt()], boots) + chosen = result.report.identity(IDENTITY).chosen + if not selected: + assert chosen.warmth is Warmth.PINNED + (entry,) = result.report.scarcity + assert entry.reason is ScarcityReason.UNPLACED + return + assert (chosen.warmth, chosen.come_alive_s, chosen.placement_wait_s) == (Warmth.COLD_WITH_BOUND, 50.0, 0.0) + assert result.report.scarcity == () + assert not [request for request in result.report.measurement_requests if request.cell == L4] + + +def test_placed_attempts_without_a_placement_wait_count_toward_the_failure_rate_and_the_percentile_alike(): + boots = [attempt(boot=50.0), *(attempt(OTHER, boot=1.0, index=index) for index in range(99))] + boots.append(unplaced(index=999)) + result = run(one(120), [receipt()], boots) + assert result.report.identity(IDENTITY).chosen.come_alive_s == 50.0 + boots = [*boots, *(unplaced(index=1000 + index) for index in range(5))] + result = run(one(0), [receipt(), receipt(cell=A100)], boots) + assert result.report.identity(IDENTITY).chosen.cell == A100 + (entry,) = result.report.scarcity + assert (entry.reason, entry.placement_failure_rate) == (ScarcityReason.UNRELIABLE_TYPE, pytest.approx(6 / 106)) + + def test_an_attempt_without_a_placement_wait_leaves_its_boot_to_stand_for_the_come_alive(): result = run(one(120), [receipt()], [attempt(boot=95.0)]) chosen = result.report.identity(IDENTITY).chosen @@ -488,7 +517,7 @@ def two_lanes(**overrides): return run(levels(**{"org/a": 0, "org/b": 0}), receipts, buffer_containers=1, **overrides) -def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): +def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_against_each_ceiling(): capacity = Capacity( total=4, types=( @@ -497,14 +526,25 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): ), shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), ) - report = two_lanes(capacity=capacity).report - assert [(use.scope, use.gpu_product, use.ceiling, use.accelerators, use.lanes) for use in report.capacity] == [ - (CapacityScope.TOTAL, None, 4, 4, ("org-a-l4", "org-b-a100-80gb")), - (CapacityScope.TYPE, A100.gpu_product, 2, 2, ("org-b-a100-80gb",)), - (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + report = two_lanes(capacity=capacity, max_containers=2).report + assert [ + (use.scope, use.gpu_product, use.ceiling, use.accelerators, use.peak_accelerators, use.lanes) + for use in report.capacity + ] == [ + (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), + (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), + (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) + assert report.to_dict()["capacity"][0] == { + "scope": "total", + "ceiling": 4, + "accelerators": 4, + "peakAccelerators": 4, + "lanes": ["org-a-l4", "org-b-a100-80gb"], + "exceeded": False, + } assert report.to_dict()["capacity"][3] == { "scope": "shared-quota", "ceiling": 6, @@ -520,11 +560,15 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): [ ( Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=1),)), - "the envelope maxima of org-a-l4 hold 2 NVIDIA-L4 accelerators, above the type ceiling of 1", + "the container ceilings of org-a-l4 hold 4 NVIDIA-L4 accelerators, above the type ceiling of 1", ), ( Capacity(total=3), - "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, above the total ceiling of 3", + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 3", + ), + ( + Capacity(total=4), + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 4", ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), @@ -533,7 +577,7 @@ def test_capacity_reports_what_the_envelope_maxima_hold_against_each_ceiling(): ), ], ) -def test_a_plan_whose_envelope_maxima_exceed_a_ceiling_is_refused_and_says_why(capacity, message): +def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert isinstance(caught.value, PlannerInputError) @@ -550,7 +594,19 @@ def test_every_exceeded_ceiling_is_named(): with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL, CapacityScope.TYPE] - assert str(caught.value).count("the envelope maxima") == 2 + assert str(caught.value).count("the container ceilings") == 2 + + +def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): + capacity = Capacity(total=4, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ + (CapacityScope.TOTAL, 4, 8), + (CapacityScope.TYPE, 2, 4), + ] + result = two_lanes(capacity=capacity, max_containers=2) + assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] def test_a_candidate_list_lane_counts_against_every_member_type(): @@ -560,31 +616,36 @@ def test_a_candidate_list_lane_counts_against_every_member_type(): TypeCeiling(gpu_product=A100.gpu_product, accelerators=1), ) ) - result = run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) - assert [use.accelerators for use in result.report.capacity] == [1, 1] + boots = [*scarce(L4), *scarce(A100)] + result = run(one(0), [receipt(), receipt(cell=A100)], boots, capacity=capacity, max_containers=1) + assert [(use.accelerators, use.peak_accelerators) for use in result.report.capacity] == [(1, 1), (1, 1)] capacity = Capacity(types=(TypeCeiling(gpu_product=A100.gpu_product, accelerators=0),)) with pytest.raises(CapacityExceededError, match="hold 1 NVIDIA-A100-SXM4-80GB accelerators"): - run(one(0), [receipt(), receipt(cell=A100)], [*scarce(L4), *scarce(A100)], capacity=capacity) + run(one(0), [receipt(), receipt(cell=A100)], boots, capacity=capacity, max_containers=1) def test_each_container_holds_its_gpu_count(): - result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0) - assert [use.accelerators for use in result.report.capacity] == [2] + result = run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=2), buffer_containers=0, max_containers=1) + assert [(use.accelerators, use.peak_accelerators) for use in result.report.capacity] == [(2, 2)] with pytest.raises(CapacityExceededError, match="hold 2 accelerators"): - run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1)) + run(one(0), [receipt(shape=PAIR)], capacity=Capacity(total=1), max_containers=1) -def test_a_zero_floor_lane_holds_nothing_until_its_envelope_may_raise_it(): - result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=0)) +def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_ceiling(): + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=4)) assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND - assert [(use.accelerators, use.lanes) for use in result.report.capacity] == [(0, ())] + assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ + (0, 4, ("org-model-l4",)) + ] + with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): + run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=3)) def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] - capacity = Capacity(types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () @@ -761,6 +822,13 @@ def failure_rate(product): held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere + if use.scope is CapacityScope.SHARED_QUOTA: + assert use.peak_accelerators is None + assert use.exceeded is (use.accelerators > use.ceiling) + else: + peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) + assert use.peak_accelerators == peak + assert use.exceeded is (peak > use.ceiling) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From 62481f7954e2e5d9d94e3e1d5a442b7011f00150 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 18:45:39 +0100 Subject: [PATCH 15/40] fix(placement): hold joined lanes to their memory and ask the right resident for evidence - Identities that join a lane in one run must fit it together. Each lazy join was checked only beside the residents the lane had before, so two joiners that fit alone could overfill the card. Joiners that do not fit together are admitted first fit in decreasing footprint and the rest are decided again without that lane. - A missing footprint or working memory is requested from the resident that lacks it, under its own identity and warmth at the lane, instead of from the joining identity. A kept lazy resident whose lane lacks memory evidence now asks for it, a profile without gpuMemoryMiB is named in the report as gpuMemoryUndeclared, and a repeated request is listed once. - Packing makes a lazy resident only on a lane where a never-cold identity is pinned, as the lazy-resident posture requires, and visits a lazy resident that is alone on its own lane like any other identity alone on its lane. - The unshared parameter names identities that never share a lane: they are never packed or made lazy residents, and no identity joins a lane they are on. - The random-estate property test also starts from current plans that are not the planner's own fixpoint, and a new property test checks that guests joining one host lane fit it together. --- packages/sie_placement/README.md | 83 +++-- .../src/sie_placement/planner.py | 327 +++++++++++++----- .../sie_placement/src/sie_placement/report.py | 2 + .../fixtures/planner/expected-report.json | 25 +- packages/sie_placement/tests/test_packing.py | 273 +++++++++++++-- packages/sie_placement/tests/test_planner.py | 35 +- 6 files changed, 589 insertions(+), 156 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 5ede7323d..29b5b6adc 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -155,8 +155,20 @@ than by a receipt of it alone: one whose `coResidents` include every other identity pinned on the lane, since a receipt measured beside more residents stands for fewer. Its warm bounds are checked and its demand is priced at that receipt. The planner never places a generation identity, one that declares -`ttftP99Ms` and `itlP99Ms` or has a generate receipt, beside another identity, -nor another identity on a lane that hosts one. +`ttftP99Ms` and `itlP99Ms` or has a generate receipt, or an identity that the +`unshared` parameter names, beside another identity, nor another identity on a +lane that hosts one. Such an identity keeps a current lane it already shares. + +Every lane that an identity joins in a run must fit all of its residents in +the plan that run writes, not only each joining identity beside the residents +it had before. Where the identities that join a lane do not fit it together, +they are admitted first fit in decreasing order of footprint, ties broken by +identity, and the others are decided again without that lane. + +Where a resident's footprint or working memory is not on file, the planner +asks that resident for it, under its own identity and warmth at the lane, in a +`footprint` measurement request. A lane on a profile without `gpuMemoryMiB` +hosts no identity beside another, and the report names that profile. ## Envelope @@ -207,6 +219,7 @@ choice, such as its own previous plan, reports no change. | `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | | `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | +| `unshared` | Identities that never share a lane: none is packed or made a lazy resident beside another identity, and no other identity joins a lane one of them is on. Empty by default | ### Selection @@ -255,9 +268,9 @@ document: it. The host must fit the identity beside the residents of its current lane, as [Several models on one GPU](#several-models-on-one-gpu) states; where a footprint or a working memory is not on file the option is a - measurement request for it. A lane with fallback cells, or one that hosts a - generation identity, hosts no lazy resident, and a generation identity is - never one. + measurement request for it, asked of each resident that lacks it. A lane + with fallback cells, or one that hosts a generation or `unshared` identity, + hosts no lazy resident, and neither identity is ever one. 5. A GPU product is unreliable when more than 5% of the cold attempts on it were `unplaced`, counting the environment's own attempts in place of borrowed ones. For an identity whose `comeAliveP95S` is at most @@ -316,22 +329,26 @@ plan total does not fall. 2. Identities that left a current lane return to it, one at a time and then together, wherever their return does not raise the plan total. An identity whose current placement is known to miss a bound does not return. -3. Packing. The planner visits every placed identity that is alone on its lane - and is not a generation identity, in decreasing order of its lane's floor - cost, ties broken by identity. For each it tries the other lanes of the plan - on its queue pool and bundle that keep a floor and host a pinned identity, - where it has a receipt at the lane's cell, container shape and concurrency: - first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, - since nothing moves that identity, then in decreasing order of the footprint - already on them, then by name. On the first lane where a move fits and lowers - the plan total by at least a cent, the identity joins it, as a lazy resident - when its bound admits its measured load onto the lane and selection did not - already offer it the lane, otherwise pinned, which needs every pinned - identity's co-residency receipt beside the others to meet its warm bounds. A - move that would lower the total but lacks a footprint, a working memory, a - load or a co-residency receipt is not made: it is a measurement request whose - `savingBound` is that fall. -4. Step 1 runs again for the lanes packing changed. +3. The identities that joined a lane are held to fitting it together, as + [Several models on one GPU](#several-models-on-one-gpu) states, and step 1 + runs again for every identity decided again. +4. Packing. The planner visits every placed identity that is alone on its lane + and is neither a generation nor an `unshared` identity, in decreasing order + of its lane's floor cost, ties broken by identity. For each it tries the + other lanes of the plan on its queue pool and bundle that keep a floor and + host a pinned identity, where it has a receipt at the lane's cell, container + shape and concurrency: first the lanes on which an identity whose + `comeAliveP95S` is `0` is pinned, since nothing moves that identity, then in + decreasing order of the footprint already on them, then by name. On the + first lane where a move fits and lowers the plan total by at least a cent, + the identity joins it, as a lazy resident when an identity whose + `comeAliveP95S` is `0` is pinned on the lane, its bound admits its measured + load onto the lane and selection did not already offer it the lane, + otherwise pinned, which needs every pinned identity's co-residency receipt + beside the others to meet its warm bounds. A move that would lower the total + but lacks a footprint, a working memory, a load or a co-residency receipt is + not made: it is a measurement request whose `savingBound` is that fall. +5. Steps 1 and 3 run again for the lanes packing changed. ### Pricing @@ -433,14 +450,19 @@ The report renders as a `placement-report` document: reason `failed`, or `unplaced` when no attempt got a container or the product's placement wait falls on an unplaced attempt, and a host lane without the identity's load asks for `lazy-resident` boot evidence on it, bounded by - what that option would cost, with `footprint` also asked for where a - resident's footprint or working memory is not on file. A packing that lacks - evidence asks at the lane it would join for `boot-evidence`, `footprint` or - `co-residency-receipt`, with the `coResidents` the receipt is to be measured - beside, bounded by the fall in the plan total the packing would bring. A kept - identity pinned beside others asks at its lane for a `co-residency-receipt` - beside them. Savings are measured against the choice, or against a kept - identity's current placement. A request that cannot save is dropped, and the + what that option would cost, with `footprint` asked of each resident whose + footprint or working memory is not on file, under that resident's identity + and warmth. A packing that lacks evidence asks at the lane it would join for + `boot-evidence`, `footprint` or `co-residency-receipt`, with the + `coResidents` the receipt is to be measured beside, bounded by the fall in + the plan total the packing would bring, and asks a resident there for its + own missing `footprint` the same way. A kept identity pinned beside others + asks at its lane for a `co-residency-receipt` beside them, and a kept lazy + resident asks each resident of its lane, itself included, for a missing + `footprint`. A request repeated for the same identity, cell, container + shape, concurrency, lane, measurements, warmth and `coResidents` is listed + once, at its highest rank. Savings are measured against the choice, or + against a kept identity's current placement. A request that cannot save is dropped, and the requests of an identity without a placement or a current one carry no bound and rank first. The evidence a kept current placement lacks is requested at its lane, named in `lane` with its `concurrency`, and also ranks first without @@ -473,6 +495,9 @@ The report renders as a `placement-report` document: plan claims that bound as met. - `warmBoundUndeclared`: every identity that has a cell and declares neither `warmP99Ms` nor `ttftP99Ms` and `itlP99Ms`, so keeps its current class. +- `gpuMemoryUndeclared`: every machine profile without `gpuMemoryMiB` on which + the planner weighed an identity beside another, so its lanes host no + identity beside another and a lane that already shares one is not checked. - `refusals`, and `changes`: the identities and lanes that differ from `current_plan`. diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 03bb6424d..14e3690b0 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -103,6 +103,7 @@ class PlannerParameters: queue_pool: str = "default" bundle: str = "default" bundles: Mapping[str, str] = field(default_factory=dict) + unshared: frozenset[str] = frozenset() def __post_init__(self) -> None: _require_name("placement", self.placement) @@ -118,6 +119,8 @@ def __post_init__(self) -> None: for identity, bundle in self.bundles.items(): _require_name("bundles", identity) _require_name(f"bundles[{identity!r}]", bundle) + for identity in self.unshared: + _require_name("unshared", identity) @property def active_floor(self) -> int: @@ -140,6 +143,8 @@ def to_dict(self) -> dict[str, object]: } if self.bundles: document["bundles"] = dict(sorted(self.bundles.items())) + if self.unshared: + document["unshared"] = sorted(self.unshared) return document @@ -331,6 +336,7 @@ class _Layout: type _CoKey = tuple[str, Cell, ContainerShape, int] +type _Packing = tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...], tuple[tuple[str, Warmth], ...]] @dataclass(frozen=True, slots=True, kw_only=True) @@ -581,6 +587,8 @@ def __init__( self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} self.co_receipts: defaultdict[_CoKey, list[tuple[str, Receipt]]] = defaultdict(list) + self.excluded: set[tuple[str, str]] = set() + self.gpu_memory_undeclared: set[str] = set() self.receipts = self._accept_receipts(receipts) self.placements = _placement_index( record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None @@ -597,10 +605,11 @@ def __init__( self.working[key] = max(receipt.working_memory_mib, self.working.get(key, 0)) for record in accepted: self._footprint(record.identity, record.cell, record.footprint_mib) - self.generation = {identity for identity, level in self.levels.items() if level.is_generation} - self.generation.update( + self.solitary = {identity for identity, level in self.levels.items() if level.is_generation} + self.solitary.update( receipt.identity for receipt in measured if receipt.request_shape.operation is Operation.GENERATE ) + self.solitary.update(parameters.unshared) def _footprint(self, identity: str, cell: Cell, footprint: int | None) -> None: if footprint is not None: @@ -687,20 +696,17 @@ def co_receipt(self, identity: str, lane: PlanLane, beside: frozenset[str]) -> t default=None, ) - def hosts_generation(self, identities: Iterable[str]) -> bool: - return any(identity in self.generation for identity in identities) + def hosts_solitary(self, identities: Iterable[str]) -> bool: + return any(identity in self.solitary for identity in identities) def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> _Memory: pinned, lazy = tuple(sorted(pinned)), tuple(sorted(lazy)) price = self.prices.get(lane.cell.machine_profile) residents = (*pinned, *lazy) - if ( - not residents - or price is None - or price.gpu_memory_mib is None - or lane.container_shape.gpu_count != 1 - or lane.fallback_cells - ): + if not residents or price is None or lane.container_shape.gpu_count != 1 or lane.fallback_cells: + return _UNPACKABLE + if price.gpu_memory_mib is None: + self.gpu_memory_undeclared.add(price.name) return _UNPACKABLE limit = price.gpu_memory_mib * MEMORY_HEADROOM_FRACTION config = (lane.cell, lane.container_shape, lane.concurrency) @@ -716,6 +722,38 @@ def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> ) return _Memory(fits=need <= limit, need=need, limit=limit) + def lane_memory(self, lane: PlanLane, residents: Mapping[str, Warmth]) -> _Memory: + return self.memory( + lane, + pinned=(identity for identity, warmth in residents.items() if warmth is Warmth.PINNED), + lazy=(identity for identity, warmth in residents.items() if warmth is Warmth.LAZY_RESIDENT), + ) + + def memory_gaps(self, lane: PlanLane, residents: Iterable[str]) -> tuple[str, ...]: + config = (lane.cell, lane.container_shape, lane.concurrency) + return tuple( + identity + for identity in sorted(set(residents)) + if (identity, lane.cell) not in self.footprints or (identity, *config) not in self.working + ) + + @staticmethod + def footprint_request( + identity: str, lane: PlanLane, warmth: Warmth, cost: float, saving: float | None = None + ) -> MeasurementRequest: + return MeasurementRequest( + identity=identity, + cell=lane.cell, + container_shape=lane.container_shape, + concurrency=lane.concurrency, + lane=lane.name, + measure=(Measurement.FOOTPRINT,), + warmth=warmth, + reason=RequestReason.UNMEASURED, + cost_bound=cost, + saving_bound=saving, + ) + def hourly(self, receipt: Receipt) -> float: return container_hourly_cost( self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier @@ -786,9 +824,10 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: lane for name, lane in sorted(self.current_lanes.items()) if (lazy if name == home else self.floor_paid(name, identity)) + and (identity, name) not in self.excluded and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells - and not self.hosts_generation((*self.current_pinned[name], *self.current_lazy[name])) + and not self.hosts_solitary((*self.current_pinned[name], *self.current_lazy[name])) and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -848,7 +887,7 @@ def current_gaps( missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) if entry.warmth in BOOT_WARMTHS and (identity, lane.cell, lane.container_shape, entry.warmth) not in self.boots: missing.append((Measurement.BOOT_EVIDENCE, RequestReason.UNMEASURED)) - return tuple( + requests = [ MeasurementRequest( identity=identity, cell=lane.cell, @@ -863,7 +902,18 @@ def current_gaps( co_residents=tuple(sorted(beside)) if measurement is Measurement.CO_RESIDENCY_RECEIPT else (), ) for measurement, reason in missing - ) + ] + if entry.warmth is Warmth.LAZY_RESIDENT: + pinned = self.current_pinned[lane.name] - {identity} + lazy = self.current_lazy[lane.name] | {identity} + if self.memory(lane, pinned=pinned, lazy=lazy).fits is None: + requests.extend( + self.footprint_request( + resident, lane, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT, cost + ) + for resident in self.memory_gaps(lane, (*pinned, *lazy)) + ) + return tuple(requests) def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice: beside = self.pinned_beside(identity) if beside is None else beside @@ -872,10 +922,10 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice options: list[_Option] = [] withdrawn: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} - load_requests: dict[str, MeasurementRequest] = {} + load_requests: dict[tuple[str, str], MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) - hosts = () if identity in self.generation else self.lazy_hosts(identity) + hosts = () if identity in self.solitary else self.lazy_hosts(identity) for order, receipt in receipts: hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second @@ -948,36 +998,49 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice continue lazy_cost = hourly * self.resident_hours(host, identity, requests, rate) load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) - memory = self.memory( - host, - pinned=self.current_pinned[host.name] - {identity}, - lazy=self.current_lazy[host.name] | {identity}, - ) + pinned = self.current_pinned[host.name] - {identity} + lazy = self.current_lazy[host.name] | {identity} + memory = self.memory(host, pinned=pinned, lazy=lazy) if memory.fits is False: continue + lacking = () if memory.fits else self.memory_gaps(host, (*pinned, *lazy)) missing = ( *(() if load is not None and load.seconds is not None else (Measurement.BOOT_EVIDENCE,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), ) - if missing: - known = load_requests.get(host.name) - if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): - load_requests[host.name] = MeasurementRequest( - identity=identity, - cell=host.cell, - container_shape=host.container_shape, - concurrency=host.concurrency, - lane=host.name, - measure=missing, - warmth=Warmth.LAZY_RESIDENT, - reason=( - RequestReason.FAILED - if load is not None and load.seconds is None - else RequestReason.UNMEASURED - ), - cost_bound=lazy_cost, - saving_bound=None, + if missing or lacking: + if blocked or level.come_alive_p95_s == 0: + continue + found = [ + self.footprint_request( + resident, host, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT, lazy_cost + ) + for resident in lacking + if resident != identity + ] + if missing: + found.append( + MeasurementRequest( + identity=identity, + cell=host.cell, + container_shape=host.container_shape, + concurrency=host.concurrency, + lane=host.name, + measure=missing, + warmth=Warmth.LAZY_RESIDENT, + reason=( + RequestReason.FAILED + if load is not None and load.seconds is None + else RequestReason.UNMEASURED + ), + cost_bound=lazy_cost, + saving_bound=None, + ) ) + for request in found: + known = load_requests.get((host.name, request.identity)) + if known is None or lazy_cost < known.cost_bound: + load_requests[(host.name, request.identity)] = request continue if load is None or load.seconds is None: continue @@ -1016,22 +1079,13 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice held = cost unpriced = cost is None gaps = self.current_gaps(identity, entry, lane, cost, beside) - gapped = { - (request.cell, request.container_shape, request.warmth) - for request in gaps - if Measurement.BOOT_EVIDENCE in request.measure - } return _Choice( identity=identity, level=level, requests=requests, options=tuple(options), chosen=chosen, - boot_requests=tuple( - request - for request in (*boot_requests.values(), *load_requests.values()) - if (request.cell, request.container_shape, request.warmth) not in gapped - ), + boot_requests=_ungapped(identity, (*boot_requests.values(), *load_requests.values()), gaps), has_valid_receipt=bool(receipts), held=held, gaps=gaps, @@ -1523,22 +1577,13 @@ def stay(self, choice: _Choice) -> _Choice | None: return None cost = self.current_cost(identity, entry, lane) gaps = self.current_gaps(identity, entry, lane, cost, choice.beside) - gapped = { - (request.cell, request.container_shape, request.warmth) - for request in gaps - if Measurement.BOOT_EVIDENCE in request.measure - } return replace( choice, chosen=None, held=cost, unpriced=cost is None, gaps=gaps, - boot_requests=tuple( - request - for request in choice.boot_requests - if (request.cell, request.container_shape, request.warmth) not in gapped - ), + boot_requests=_ungapped(identity, choice.boot_requests, gaps), ) @staticmethod @@ -1589,13 +1634,14 @@ def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, f (lane, frozenset(pinned[lane.name]), frozenset(lazy[lane.name])) for lane in layout.plan.lanes if lane.name != layout.names[identity] + and (identity, lane.name) not in self.excluded and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and lane.container_shape.gpu_count == 1 and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and pinned[lane.name] - and not self.hosts_generation((*pinned[lane.name], *lazy[lane.name])) + and not self.hosts_solitary((*pinned[lane.name], *lazy[lane.name])) and self.slowest_receipt(identity, lane) is not None ] return sorted( @@ -1613,7 +1659,7 @@ def packings( target: tuple[PlanLane, frozenset[str], frozenset[str]], layout: _Layout, choices: Mapping[str, _Choice], - ) -> list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]]: + ) -> list[_Packing]: lane, pinned, lazy = target level = self.levels[identity] order, receipt = min( @@ -1625,7 +1671,7 @@ def packings( offered = choices[identity] hosts = {option.host for option in offered.options if option.warmth is Warmth.LAZY_RESIDENT} hosts.update(request.lane for request in offered.boot_requests if request.warmth is Warmth.LAZY_RESIDENT) - packings: list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]] = [] + packings: list[_Packing] = [] current_class = self.current_class(identity) blocked = (*self.warm_blockers(level, receipt, current_class), *self.reliability_blockers(level, lane.cells)) load = self.boots.get((identity, lane.cell, lane.container_shape, Warmth.LAZY_RESIDENT)) @@ -1633,14 +1679,16 @@ def packings( level.come_alive_p95_s != 0 and not blocked and lane.name not in hosts + and any(self.levels[resident].come_alive_p95_s == 0 for resident in pinned) and (load is None or load.seconds is not None) and (load is None or load.seconds is None or _come_alive_blocker(level, load.seconds) is None) ): memory = self.memory(lane, pinned=pinned, lazy=lazy | {identity}) if memory.fits is not False: + lacking = () if memory.fits else self.memory_gaps(lane, (*pinned, *lazy, identity)) missing = ( *(() if load is not None else (Measurement.BOOT_EVIDENCE,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), ) option = _Option( receipt=receipt, @@ -1655,7 +1703,12 @@ def packings( host=host, host_owner=owner, ) - packings.append(({identity: option}, missing, ())) + others = tuple( + (resident, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT) + for resident in lacking + if resident != identity + ) + packings.append(({identity: option}, missing, (), others)) members = pinned | {identity} if all(choices[member].chosen is not None for member in pinned): together = {member: self.co_receipt(member, lane, members - {member}) for member in members} @@ -1669,9 +1722,15 @@ def packings( ): memory = self.memory(lane, pinned=members, lazy=lazy) if memory.fits is not False: + lacking = () if memory.fits else self.memory_gaps(lane, (*members, *lazy)) missing = ( *(() if len(measured) == len(members) else (Measurement.CO_RESIDENCY_RECEIPT,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), + ) + others = tuple( + (resident, Warmth.PINNED if resident in members else Warmth.LAZY_RESIDENT) + for resident in lacking + if resident != identity ) own_order, own = measured.get(identity, (order, receipt)) updates = { @@ -1688,7 +1747,7 @@ def packings( host_owner=owner, ) } - if not missing: + if not missing and not others: for member in sorted(pinned): chosen = choices[member].chosen member_order, member_receipt = measured[member] @@ -1701,7 +1760,7 @@ def packings( cost=self.hourly(member_receipt) * self.window.warm_container_hours(choices[member].requests, rate), ) - packings.append((updates, missing, tuple(sorted(pinned)))) + packings.append((updates, missing, tuple(sorted(pinned)), others)) return packings def pack(self, choices: dict[str, _Choice]) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: @@ -1721,19 +1780,18 @@ def floor_cost(identity: str) -> float: identity for identity, choice in choices.items() if (chosen := choice.chosen) is not None - and chosen.host is None + and chosen.host in (None, layout.names[identity]) and chosen.host_owner is None - and chosen.warmth is not Warmth.LAZY_RESIDENT - and identity not in self.generation + and identity not in self.solitary and hosted[layout.names[identity]] == 1 ] - requests: dict[tuple[str, str, Warmth], MeasurementRequest] = {} + requests: dict[tuple[str, str | None, Warmth, tuple[Measurement, ...]], MeasurementRequest] = {} for identity in sorted(candidates, key=lambda identity: (-floor_cost(identity), identity)): if hosted.get(layout.names.get(identity, ""), 0) != 1: continue moved = False for target in self.pack_targets(identity, layout): - for updates, missing, beside in self.packings(identity, target, layout, choices): + for updates, missing, beside, lacking in self.packings(identity, target, layout, choices): members = frozenset(beside) | {identity} trial = { **choices, @@ -1750,23 +1808,32 @@ def floor_cost(identity: str) -> float: if round(saving, 2) <= 0: continue option = updates[identity] - if missing: - key = (identity, target[0].name, option.warmth) - known = requests.get(key) - if known is None or saving > (known.saving_bound or 0.0): - requests[key] = MeasurementRequest( - identity=identity, - cell=target[0].cell, - container_shape=target[0].container_shape, - concurrency=target[0].concurrency, - lane=target[0].name, - measure=missing, - warmth=option.warmth, - reason=RequestReason.UNMEASURED, - cost_bound=option.cost, - saving_bound=saving, - co_residents=beside, + if missing or lacking: + found = [ + self.footprint_request(resident, target[0], warmth, option.cost, saving) + for resident, warmth in lacking + ] + if missing: + found.append( + MeasurementRequest( + identity=identity, + cell=target[0].cell, + container_shape=target[0].container_shape, + concurrency=target[0].concurrency, + lane=target[0].name, + measure=missing, + warmth=option.warmth, + reason=RequestReason.UNMEASURED, + cost_bound=option.cost, + saving_bound=saving, + co_residents=beside, + ) ) + for request in found: + key = (request.identity, request.lane, request.warmth, request.measure) + known = requests.get(key) + if known is None or saving > (known.saving_bound or 0.0): + requests[key] = request continue choices, layout, total = trial, trial_layout, trial_total hosted = Counter(entry.lane for entry in layout.plan.identities) @@ -1801,13 +1868,56 @@ def settle(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: choices = {**choices, **{identity: self.choose(identity, beside) for identity, beside in stale.items()}} return choices + def overflowing(self, choices: Mapping[str, _Choice]) -> list[tuple[str, str]]: + layout = self.assemble(choices) + before: defaultdict[str, set[str]] = defaultdict(set) + if self.current_plan is not None: + for entry in self.current_plan.identities: + before[entry.lane].add(entry.identity) + residents: defaultdict[str, dict[str, Warmth]] = defaultdict(dict) + for entry in layout.plan.identities: + residents[entry.lane][entry.identity] = entry.warmth + rejected: list[tuple[str, str]] = [] + for lane in layout.plan.lanes: + hosted = residents[lane.name] + joiners = sorted( + ( + identity + for identity in hosted + if identity not in before[lane.name] + and (chosen := choices[identity].chosen) is not None + and (chosen.host is not None or chosen.host_owner is not None) + ), + key=lambda identity: (-self.footprints.get((identity, lane.cell), 0), identity), + ) + if not joiners or self.lane_memory(lane, hosted).fits: + continue + admitted = {identity: warmth for identity, warmth in hosted.items() if identity not in joiners} + for identity in joiners: + trial = {**admitted, identity: hosted[identity]} + if self.lane_memory(lane, trial).fits: + admitted = trial + else: + rejected.append((identity, lane.name)) + return rejected + + def fit(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: + for _ in range(len(self.levels) * (len(self.current_lanes) + len(self.levels)) + 1): + choices = self.settle(choices) + rejected = self.overflowing(choices) + if not rejected: + return choices + self.excluded.update(rejected) + choices = {**choices, **{identity: self.choose(identity) for identity, _ in rejected}} + return self.settle(choices) + def decide(self) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) originals = {identity: self.choose(identity) for identity in sorted(self.levels)} - decided = self.stay_beside(self.settle(dict(originals)), originals) + decided = self.fit(self.stay_beside(self.settle(dict(originals)), originals)) decided, requests = self.pack(decided) - return self.settle(decided), requests + return self.fit(decided), requests def run(self, *, price_savings: bool = True) -> PlannerResult: decided, packing_requests = self.decide() @@ -1843,7 +1953,7 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: identities=tuple(reports), lanes=lane_reports, shared_chart_lanes=_shared_chart_lanes(new_plan), - measurement_requests=tuple(sorted(requests, key=_request_rank)), + measurement_requests=_distinct(requests), come_alive_levers=tuple(sorted(levers, key=_lever_rank)), queue_wait_unattained=tuple( identity for identity in sorted(self.levels) if self.levels[identity].queue_wait_p99_ms is not None @@ -1858,6 +1968,7 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: ), capacity=self.capacity_use(new_plan), scarcity=tuple(sorted(scarcity, key=_scarcity_rank)), + gpu_memory_undeclared=tuple(sorted(self.gpu_memory_undeclared)), ) return PlannerResult(plan=new_plan, report=report) @@ -1951,6 +2062,38 @@ def _pricing( return tuple(reports), costs +def _ungapped( + identity: str, requests: Iterable[MeasurementRequest], gaps: Iterable[MeasurementRequest] +) -> tuple[MeasurementRequest, ...]: + gapped = { + (request.cell, request.container_shape, request.warmth) + for request in gaps + if request.identity == identity and Measurement.BOOT_EVIDENCE in request.measure + } + return tuple( + request + for request in requests + if request.identity != identity or (request.cell, request.container_shape, request.warmth) not in gapped + ) + + +def _distinct(requests: Iterable[MeasurementRequest]) -> tuple[MeasurementRequest, ...]: + kept: dict[tuple[object, ...], MeasurementRequest] = {} + for request in sorted(requests, key=_request_rank): + key = ( + request.identity, + request.cell, + request.container_shape, + request.concurrency, + request.lane, + request.measure, + request.warmth, + request.co_residents, + ) + kept.setdefault(key, request) + return tuple(kept.values()) + + def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | None: fields = _fields(option.blockers) if fields == [PLACEMENT_FAILURE_FIELD]: diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 430fdcda0..338d50f97 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -428,6 +428,7 @@ class Report: warm_bound_undeclared: tuple[str, ...] = () capacity: tuple[CapacityUse, ...] = () scarcity: tuple[Scarcity, ...] = () + gpu_memory_undeclared: tuple[str, ...] = () @property def monthly_cost(self) -> float: @@ -472,6 +473,7 @@ def to_dict(self) -> dict[str, object]: "scarcity": [entry.to_dict() for entry in self.scarcity], "queueWaitUnattained": list(self.queue_wait_unattained), "warmBoundUndeclared": list(self.warm_bound_undeclared), + "gpuMemoryUndeclared": list(self.gpu_memory_undeclared), "refusals": [refusal.to_dict() for refusal in self.refusals], "changes": { "identities": [change.to_dict() for change in self.identity_changes], diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 91a4f0861..226647ae1 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -482,13 +482,33 @@ "concurrency": 16, "lane": "default-l4-default", "measure": [ - "boot-evidence", - "footprint" + "boot-evidence" ], "warmth": "lazy-resident", "reason": "unmeasured", "costBound": 0.0, "savingBound": 47.45 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "lane": "default-l4-default", + "measure": [ + "footprint" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 0.0, + "savingBound": 47.45 } ], "comeAliveLevers": [ @@ -540,6 +560,7 @@ "intfloat/e5-base-v2" ], "warmBoundUndeclared": [], + "gpuMemoryUndeclared": [], "refusals": [ { "record": "boot-evidence", diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index 2449df044..ae04e6feb 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -276,15 +276,85 @@ def test_a_packing_without_footprints_is_a_measurement_request_with_its_saving_b assert requests[1].to_dict()["measure"] == ["footprint"] -def test_a_packing_without_a_load_asks_for_the_load_and_the_footprints_it_lacks(): +def test_a_packing_asks_the_joiner_for_its_load_and_each_resident_for_the_memory_evidence_it_lacks(): receipts = [receipt(HOST), receipt(GUEST, footprint=4000, working=1500)] result = run(levels(**{HOST: 0, GUEST: 60}), receipts) - lazy = [ - request + on_host = [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] + assert [(request.identity, request.warmth, request.measure) for request in on_host] == [ + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.BOOT_EVIDENCE,)), + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT,)), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,)), + ] + assert all(request.saving_bound == pytest.approx(MONTH) for request in on_host) + + +def test_a_host_resident_without_memory_evidence_is_asked_for_it_under_its_own_identity(): + current = plan_of([lane("host"), lane("own")], {HOST: ("host", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + receipts = [receipt(HOST), receipt(GUEST, footprint=1000, working=500)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)], current=current) + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + on_host = [request for request in result.report.measurement_requests if request.lane == "host"] + assert [(request.identity, request.warmth, request.measure) for request in on_host] == [ + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT,)), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,)), + ] + assert on_host[1].saving_bound == pytest.approx(MONTH) + + +def test_a_resident_several_joiners_need_measured_is_asked_once(): + current = plan_of( + [lane("host"), lane("own-a"), lane("own-b")], + {HOST: ("host", Warmth.PINNED), "org/a": ("own-a", Warmth.PINNED), "org/b": ("own-b", Warmth.PINNED)}, + ) + receipts = [ + receipt(HOST), + receipt("org/a", footprint=1000, working=500), + receipt("org/b", footprint=1000, working=500), + ] + result = run( + levels(**{HOST: 0, "org/a": 60, "org/b": 60}), receipts, [load("org/a"), load("org/b")], current=current + ) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + "org/a": ("own-a", Warmth.PINNED), + "org/b": ("own-b", Warmth.PINNED), + } + asked = [request for request in result.report.measurement_requests if request.lane == "host"] + assert [(request.identity, request.measure, request.warmth) for request in asked] == [ + ("org/a", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED), + ("org/b", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED), + (HOST, (Measurement.FOOTPRINT,), Warmth.PINNED), + ] + assert asked[2].saving_bound == pytest.approx(MONTH) + + +@pytest.mark.parametrize("price_list", [prices(), prices(cards=False)], ids=["no-footprints", "no-card-memory"]) +def test_a_current_lazy_resident_whose_lane_lacks_memory_evidence_is_kept_and_says_what_is_missing(price_list): + current = plan_of([lane("host")], {HOST: ("host", Warmth.PINNED), GUEST: ("host", Warmth.LAZY_RESIDENT)}) + result = run( + levels(**{HOST: 0, GUEST: 60}), + [receipt(HOST), receipt(GUEST)], + [load(GUEST)], + current=current, + price_list=price_list, + ) + assert result.plan == current + report = result.report.identity(GUEST) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + requests = [ + (request.identity, request.warmth, request.measure, request.saving_bound) for request in result.report.measurement_requests - if request.lane == "org-host-l4" and request.warmth is Warmth.LAZY_RESIDENT ] - assert [request.measure for request in lazy] == [(Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)] + if price_list.machine_profiles[0].gpu_memory_mib is None: + assert requests == [] + assert result.report.gpu_memory_undeclared == ("l4",) + assert result.report.to_dict()["gpuMemoryUndeclared"] == ["l4"] + return + assert requests == [ + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.FOOTPRINT,), None), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,), None), + ] + assert result.report.gpu_memory_undeclared == () @pytest.mark.parametrize( @@ -444,6 +514,32 @@ def test_packing_fills_the_lane_with_the_most_footprint_in_use_first(): assert ("org/h2", "org-h1-l4", Warmth.PINNED) in requests +@pytest.mark.parametrize(("bound", "joined"), [(0, True), (60, False)]) +def test_packing_makes_a_lazy_resident_only_beside_a_never_cold_pinned_identity(bound, joined): + receipts = [receipt("org/warm", footprint=9000, working=1000), receipt(GUEST, footprint=2000, working=1000)] + result = run(levels(**{"org/warm": bound, GUEST: 60}), receipts, [load(GUEST)]) + if joined: + assert placements(result)[GUEST] == ("org-warm-l4", Warmth.LAZY_RESIDENT) + return + assert placements(result) == { + "org/warm": ("org-warm-l4", Warmth.PINNED), + GUEST: ("org-guest-l4", Warmth.PINNED), + } + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + + +def test_a_lazy_resident_alone_on_its_own_lane_is_packed_beside_a_never_cold_identity_in_the_same_run(): + current = plan_of([lane("own")], {GUEST: ("own", Warmth.LAZY_RESIDENT)}) + receipts = [receipt(HOST, footprint=5000, working=500), receipt(GUEST, footprint=5000, working=500)] + service_levels = levels(**{HOST: 0, GUEST: 600}) + result = run(service_levels, receipts, [load(GUEST)], current=current) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert result.report.monthly_cost == pytest.approx(MONTH) + again = run(service_levels, receipts, [load(GUEST)], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + def test_packing_tries_lanes_a_never_cold_identity_pins_before_the_others(): receipts = [ receipt("org/a-warm", footprint=9000, working=1000), @@ -494,6 +590,67 @@ def test_the_lazy_resident_selection_on_a_current_lane_requires_the_memory_check assert requests == ([measure] if measure else []) +def joining(residents): + current = plan_of( + [lane("host"), *(lane(f"own-{identity[-1]}") for identity in residents)], + {HOST: ("host", Warmth.PINNED), **{identity: (f"own-{identity[-1]}", Warmth.PINNED) for identity in residents}}, + ) + receipts = [receipt(HOST, footprint=6000, working=1000)] + receipts.extend(receipt(identity, footprint=size, working=work) for identity, (size, work) in residents.items()) + boots = [load(identity) for identity in residents] + return levels(**{HOST: 0, **dict.fromkeys(residents, 60)}), receipts, boots, current + + +@pytest.mark.parametrize( + ("residents", "joined", "memory"), + [ + ({"org/a": (8000, 1000), "org/b": (1000, 8000)}, {"org/a"}, 6000 + 8000 + 1000), + ({"org/a": (7000, 1000), "org/b": (2000, 6000), "org/c": (3000, 1000)}, {"org/a", "org/c"}, 6000 + 7000 + 1000), + ], + ids=["two", "three"], +) +def test_lazy_residents_that_each_fit_a_lane_alone_join_it_only_as_far_as_they_fit_together(residents, joined, memory): + for identity, sizes in residents.items(): + alone = run(*joining({identity: sizes})[:3], current=joining({identity: sizes})[3]) + assert placements(alone)[identity] == ("host", Warmth.LAZY_RESIDENT) + service_levels, receipts, boots, current = joining(residents) + result = run(service_levels, receipts, boots, current=current) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + **{ + identity: ("host", Warmth.LAZY_RESIDENT) if identity in joined else (f"own-{identity[-1]}", Warmth.PINNED) + for identity in residents + }, + } + host = {lane_report.name: lane_report for lane_report in result.report.lanes}["host"] + assert (host.memory_mib, host.memory_limit_mib) == (memory, pytest.approx(LIMIT)) + again = run(service_levels, receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +@pytest.mark.parametrize("unshared", [GUEST, HOST]) +def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(unshared): + service_levels = levels(**{HOST: 0, GUEST: 60}) + result = run(service_levels, host_and_guest(), [load(GUEST)], unshared=frozenset({unshared})) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + assert result.report.parameters["unshared"] == [unshared] + current = plan_of([lane("host"), lane("own")], {HOST: ("host", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + kept = run(service_levels, host_and_guest(), [load(GUEST)], current=current, unshared=frozenset({unshared})) + assert kept.plan == current + assert all(request.lane != "host" for request in kept.report.measurement_requests) + shared = run(service_levels, host_and_guest(), [load(GUEST)], current=current) + assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) + + +def test_an_unshared_identity_keeps_a_current_shared_lane_it_already_holds(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current, unshared=frozenset({"org/a"})) + assert result.plan == current + assert result.report.unchanged + + def test_an_identity_whose_neighbours_departure_strips_its_co_residency_evidence_is_decided_again_in_the_run(): current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) receipts = [ @@ -577,9 +734,23 @@ def test_identities_that_leave_a_lane_together_return_when_sharing_it_costs_less IDENTITIES = ("org/m0", "org/m1", "org/m2", "org/m3", "org/m4") -@pytest.mark.parametrize("seed", range(150)) -def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): - rng = random.Random(seed) # noqa: S311 -- reproducible test data +def scattered(rng, identities, generation) -> Plan | None: + names = [f"current-{index}" for index in range(rng.randint(1, 3))] + residents = { + identity: (rng.choice(names), rng.choice([Warmth.PINNED, Warmth.PINNED, Warmth.LAZY_RESIDENT])) + for identity in identities + if identity not in generation and rng.random() < 0.8 + } + residents.update( + (identity, (f"alone-{identity[-1]}", Warmth.PINNED)) for identity in sorted(generation) if rng.random() < 0.8 + ) + used = sorted({name for name, _ in residents.values()}) + if not used: + return None + return plan_of([lane(name, rng.choice([L4, L4, A100])) for name in used], residents) + + +def random_estate(rng): identities = list(IDENTITIES[: rng.randint(2, 5)]) generation = {identity for identity in identities if rng.random() < 0.15} entries = [] @@ -594,8 +765,8 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): ) ) service_levels = ServiceLevels(identities=tuple(entries)) - footprints = {identity: rng.choice([None, 2000, 5000, 9000]) for identity in identities} - working = {identity: rng.choice([None, 500, 2500]) for identity in identities} + footprints = {identity: rng.choice([None, 2000, 5000, 9000, 12000]) for identity in identities} + working = {identity: rng.choice([None, 500, 2500, 8000]) for identity in identities} receipts = [] for identity in identities: for cell in (L4, A100): @@ -633,31 +804,39 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} for identity in identities } + unshared = frozenset(identity for identity in plain if rng.random() < 0.15) + draw = rng.random() current = None - if rng.random() < 0.5: - first = run(service_levels, receipts, boots, hourly=hourly) - current = first.plan - result = run(service_levels, receipts, boots, hourly=hourly, current=current) + if draw < 0.35: + current = run(service_levels, receipts, boots, hourly=hourly, unshared=unshared).plan + elif draw < 0.7: + current = scattered(rng, identities, generation) + return service_levels, receipts, boots, hourly, current, unshared, generation, not 0.35 <= draw < 0.7 + + +@pytest.mark.parametrize("seed", range(200)) +def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + service_levels, receipts, boots, hourly, current, unshared, generation, settled = random_estate(rng) + result = run(service_levels, receipts, boots, hourly=hourly, current=current, unshared=unshared) assert Plan.from_dict(result.plan.to_dict()) == result.plan levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} residents: dict[str, list[PlanIdentity]] = {} for entry in result.plan.identities: residents.setdefault(entry.lane, []).append(entry) - before = {} if current is None else {planned.name: planned for planned in current.lanes} + before: dict[str, set[str]] = {} + for entry in () if current is None else current.identities: + before.setdefault(entry.lane, set()).add(entry.identity) reports = {lane_report.name: lane_report for lane_report in result.report.lanes} for planned in result.plan.lanes: hosted = residents[planned.name] - if any(entry.identity in generation for entry in hosted): - assert len(hosted) == 1 if len(hosted) < 2: continue pinned = {entry.identity for entry in hosted if entry.warmth is Warmth.PINNED} - formed = planned.name not in before or { - entry.identity for entry in current.identities if entry.lane == planned.name - } != {entry.identity for entry in hosted} - if formed: + if {entry.identity for entry in hosted} - before.get(planned.name, set()): assert reports[planned.name].memory_mib is not None assert reports[planned.name].memory_mib <= reports[planned.name].memory_limit_mib + assert not any(entry.identity in generation | unshared for entry in hosted) for entry in hosted: report = result.report.identity(entry.identity) if report.chosen is None: @@ -672,13 +851,14 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): assert level.come_alive_p95_s is None or report.chosen.come_alive_s <= level.come_alive_p95_s for request in result.report.measurement_requests: assert request.saving_bound is None or request.saving_bound > 0 - again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan) - assert again.plan == result.plan - assert again.report.unchanged - assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + if settled: + again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan, unshared=unshared) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) shuffled = list(receipts) rng.shuffle(shuffled) - reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current) + reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current, unshared=unshared) assert rendered(reordered) == rendered(result) for entry in service_levels.identities: bound = entry.level.come_alive_p95_s @@ -690,5 +870,44 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): for other in service_levels.identities ) ) - loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan) + loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan, unshared=unshared) assert loosened.report.monthly_cost <= result.report.monthly_cost + 0.01 + + +def need(host, guests, sizes): + return ( + host[0] + + max((sizes[guest][0] for guest in guests), default=0) + + max((host[1], *(sizes[guest][1] for guest in guests))) + ) + + +@pytest.mark.parametrize("seed", range(150)) +def test_random_guests_join_a_host_lane_only_as_far_as_they_fit_together(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + host = (rng.choice([4000, 6000, 8000]), rng.choice([500, 1000, 2000])) + sizes = { + f"org/g{index}": (rng.choice([1000, 3000, 6000, 9000]), rng.choice([500, 2000, 6000, 9000])) + for index in range(rng.randint(2, 4)) + } + current = plan_of( + [lane("host"), *(lane(f"own-{guest[-1]}") for guest in sizes)], + {HOST: ("host", Warmth.PINNED), **{guest: (f"own-{guest[-1]}", Warmth.PINNED) for guest in sizes}}, + ) + receipts = [receipt(HOST, footprint=host[0], working=host[1])] + receipts.extend(receipt(guest, footprint=size, working=work) for guest, (size, work) in sizes.items()) + boots = [load(guest) for guest in sizes] + service_levels = levels(**{HOST: 0, **dict.fromkeys(sizes, 60)}) + result = run(service_levels, receipts, boots, current=current) + joined = {guest for guest, (name, _) in placements(result).items() if name == "host" and guest != HOST} + assert all(placements(result)[guest] == ("host", Warmth.LAZY_RESIDENT) for guest in joined) + assert need(host, joined, sizes) <= LIMIT + for guest in sizes.keys() - joined: + assert placements(result)[guest] == (f"own-{guest[-1]}", Warmth.PINNED) + assert need(host, joined | {guest}, sizes) > LIMIT + if joined: + (report,) = [lane_report for lane_report in result.report.lanes if lane_report.name == "host"] + assert report.memory_mib == need(host, joined, sizes) + again = run(service_levels, receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index c4bf3886c..e0d78d211 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1199,8 +1199,16 @@ def test_a_lazy_resident_without_evidence_keeps_its_place_and_asks_for_it(): ] assert requests == [ ((Measurement.BOOT_EVIDENCE,), Warmth.LAZY_RESIDENT, "shared", 8, None), + ((Measurement.FOOTPRINT,), Warmth.LAZY_RESIDENT, "shared", 8, None), ((Measurement.RECEIPT,), Warmth.LAZY_RESIDENT, "shared", 8, None), ] + (anchor,) = [request for request in result.report.measurement_requests if request.identity == ANCHOR] + assert (anchor.measure, anchor.warmth, anchor.lane, anchor.saving_bound) == ( + (Measurement.FOOTPRINT,), + Warmth.PINNED, + "shared", + None, + ) def test_a_measured_load_onto_a_lane_a_never_cold_neighbour_pins_makes_a_lazy_resident(): @@ -1335,17 +1343,22 @@ def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): assert all(request.lane != "shared" for request in result.report.measurement_requests) -def test_a_current_lane_hosts_lazy_residents_beside_a_never_cold_neighbour_and_a_plan_lane_beside_any_pinned_one(): +def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_current_lane_and_on_a_packed_one(): current = plan_of( [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} ) result = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) - assert (report.lane, report.chosen.warmth, report.chosen.lane) == ("shared", Warmth.LAZY_RESIDENT, "shared") - assert result.report.monthly_cost == pytest.approx(MONTH) + assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert result.report.monthly_cost == pytest.approx(2 * MONTH) assert all(lever.warmth is not Warmth.LAZY_RESIDENT for lever in result.report.come_alive_levers) - again = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=result.plan) - assert again.report.unchanged + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + fresh = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY) + assert {entry.identity: entry.warmth for entry in fresh.plan.identities} == { + ANCHOR: Warmth.PINNED, + IDENTITY: Warmth.PINNED, + } + assert len(fresh.plan.lanes) == 2 @pytest.mark.parametrize( @@ -1546,6 +1559,9 @@ def test_parameters_echo_into_the_report(): "bundles": {"org/a": "default", "org/b": "sglang"}, } assert list(echoed["bundles"]) == ["org/a", "org/b"] + assert parameters(unshared=frozenset({"org/b", "org/a"})).to_dict()["unshared"] == ["org/a", "org/b"] + with pytest.raises(SchemaError, match="unshared"): + parameters(unshared=frozenset({"org/a b"})) def test_the_placement_must_be_in_the_price_list(): @@ -1646,11 +1662,18 @@ def test_the_estate_report_states_its_findings(): ("intfloat/e5-base-v2", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED, "default-l4-default", None), ( "BAAI/bge-m3:sparse", - (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT), + (Measurement.BOOT_EVIDENCE,), Warmth.LAZY_RESIDENT, "default-l4-default", pytest.approx(sparse.chosen.monthly_cost), ), + ( + "intfloat/e5-base-v2", + (Measurement.FOOTPRINT,), + Warmth.PINNED, + "default-l4-default", + pytest.approx(sparse.chosen.monthly_cost), + ), ] assert dense.chosen.co_residents == ("intfloat/e5-base-v2",) assert dense.chosen.receipt.uri == "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" From 1d4809bb82aa784f94f416bfbd74b97beb9dc1ac Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:01:26 +0100 Subject: [PATCH 16/40] fix(placement): hold lanes to the shared quota at maxContainers - The shared quota now holds every lane at its container ceiling, with the accelerators held elsewhere, as the total and per-type ceilings already do. The report gives peakAccelerators for every scope, and the refusal names every lane the quota covers and the container ceilings that exceed it. - A plan that does not fit is refused whole: the refused plan keeps every lane's maxContainers, which the tests now pin. - A loosening whose new lane would exceed the quota at its container ceiling saves nothing, while the carried lane it would leave fits. --- packages/sie_placement/README.md | 20 +++---- .../src/sie_placement/planner.py | 3 +- .../sie_placement/src/sie_placement/report.py | 10 ++-- .../sie_placement/tests/test_availability.py | 53 ++++++++++++++----- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index caf70a449..4e39543e6 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -117,12 +117,14 @@ obtain. Each field is optional, and at least one is declared: | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | Every container holds the `gpuCount` accelerators of its lane's container -shape. `total` and each of `types` must hold every lane they cover at its -container ceiling, `maxContainers` containers, at once, and so at its envelope -maximum, `floorMax` containers, too. `sharedQuota` bounds the envelope maxima, -the most a tuner may hold warm, together with `heldElsewhere`. A lane with -fallback cells counts against the ceiling of every product it names, and once -against `total`. +shape. Each ceiling must hold every lane it covers at its container ceiling, +`maxContainers` containers, at once, as a deployment compiler that enforces the +ceiling charges it, and so at its envelope maximum, `floorMax` containers, too. +`total` and `sharedQuota` cover every lane, and `sharedQuota` holds them +together with its `heldElsewhere` accelerators. A lane with fallback cells +counts against the ceiling of every product it names, and once against `total` +and `sharedQuota`. The planner never lowers a lane's `maxContainers` or +`floorMax` to fit a capacity document: a plan that does not fit is refused. ## Envelope @@ -343,9 +345,9 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, for `total` and `type` the `peakAccelerators` its - lanes hold at `maxContainers`, the `lanes` that hold them, and whether it is - `exceeded`. + maxima hold against it, the `peakAccelerators` its lanes hold at + `maxContainers`, both with `heldElsewhere` for `shared-quota`, the `lanes` + that hold them, and whether it is `exceeded`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 48e5ff752..505dff824 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1176,8 +1176,9 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: scope=CapacityScope.SHARED_QUOTA, ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, + peak_accelerators=peak + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), + lanes=names, ) ) return tuple(uses) diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 29bbb4a34..6d4a81bc1 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -323,18 +323,18 @@ def exceeded(self) -> bool: def describe(self) -> str: lanes = ", ".join(self.lanes) or "no lane" + held, count = "envelope maxima", self.accelerators + if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: + held, count = "container ceilings", self.peak_accelerators if self.scope is CapacityScope.SHARED_QUOTA: elsewhere = self.held_elsewhere or 0 return ( - f"the envelope maxima of {lanes} hold {self.accelerators - elsewhere} accelerators, " + f"the {held} of {lanes} hold {count - elsewhere} accelerators, " f"which with {elsewhere} held elsewhere exceed the shared quota of {self.ceiling}" ) accelerators = "accelerators" if self.gpu_product is None else f"{self.gpu_product} accelerators" - held = f"the envelope maxima of {lanes} hold {self.accelerators}" - if self.peak_accelerators is not None and self.peak_accelerators > self.ceiling: - held = f"the container ceilings of {lanes} hold {self.peak_accelerators}" scope = "type" if self.scope is CapacityScope.TYPE else "total" - return f"{held} {accelerators}, above the {scope} ceiling of {self.ceiling}" + return f"the {held} of {lanes} hold {count} {accelerators}, above the {scope} ceiling of {self.ceiling}" def to_dict(self) -> dict[str, object]: return _compact( diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index ae226c927..0304772b6 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -534,7 +534,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.SHARED_QUOTA, None, 6, 6, 6, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) assert report.to_dict()["capacity"][0] == { @@ -549,6 +549,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a "scope": "shared-quota", "ceiling": 6, "accelerators": 6, + "peakAccelerators": 6, "heldElsewhere": 2, "lanes": ["org-a-l4", "org-b-a100-80gb"], "exceeded": False, @@ -572,9 +573,14 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), - "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " "which with 3 held elsewhere exceed the shared quota of 6", ), + ( + Capacity(shared_quota=SharedQuota(accelerators=10, held_elsewhere=3)), + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " + "which with 3 held elsewhere exceed the shared quota of 10", + ), ], ) def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): @@ -598,17 +604,37 @@ def test_every_exceeded_ceiling_is_named(): def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): - capacity = Capacity(total=4, types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),)) + capacity = Capacity( + total=4, + types=(TypeCeiling(gpu_product="NVIDIA-L4", accelerators=2),), + shared_quota=SharedQuota(accelerators=7, held_elsewhere=3), + ) with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ (CapacityScope.TOTAL, 4, 8), (CapacityScope.TYPE, 2, 4), + (CapacityScope.SHARED_QUOTA, 7, 11), ] result = two_lanes(capacity=capacity, max_containers=2) assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] +def test_a_refused_plan_keeps_every_container_ceiling_it_would_need(): + capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=5)) + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=capacity) + refused = caught.value.result.plan + assert [(planned.name, planned.max_containers) for planned in refused.lanes] == [ + ("org-a-l4", 4), + ("org-b-a100-80gb", 4), + ] + assert refused == two_lanes().plan + for use in caught.value.exceeded: + assert use.lanes == ("org-a-l4", "org-b-a100-80gb") + assert "org-a-l4, org-b-a100-80gb hold 8 accelerators" in use.describe() + + def test_a_candidate_list_lane_counts_against_every_member_type(): capacity = Capacity( types=( @@ -643,14 +669,18 @@ def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_cei def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] - free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) + current = plan_of( + [replace(lane("org-model-l4", buffer=1), max_containers=2)], {IDENTITY: ("org-model-l4", Warmth.PINNED)} + ) + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, current=current) + assert free.plan == current assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) - bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=current) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 - with pytest.raises(CapacityExceededError): + with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) @@ -820,15 +850,12 @@ def failure_rate(product): if use.scope is not CapacityScope.TYPE or use.gpu_product in {cell.gpu_product for cell in planned.cells} ] held = sum(planned.envelope.floor_max * planned.container_shape.gpu_count for planned in lanes) + peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere - if use.scope is CapacityScope.SHARED_QUOTA: - assert use.peak_accelerators is None - assert use.exceeded is (use.accelerators > use.ceiling) - else: - peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) - assert use.peak_accelerators == peak - assert use.exceeded is (peak > use.ceiling) + assert use.peak_accelerators == peak + elsewhere + assert use.exceeded is (peak + elsewhere > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From e8d0f5e6edcef77148036047e54e290118ef6592 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:03:24 +0100 Subject: [PATCH 17/40] test(placement): hold shared lanes to the ceilings at maxContainers - A shared lane holds its container ceiling once against the total, its GPU type and the shared quota, whichever residents it carries, so a packing fits where the separate lanes would be refused. - A shared lane over a ceiling is refused with its maxContainers intact, and the refusal names it. --- packages/sie_placement/tests/test_packing.py | 48 ++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index ae04e6feb..b3287c54f 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -5,11 +5,12 @@ from datetime import UTC, datetime, timedelta import pytest -from sie_placement.planner import PlannerParameters, plan +from sie_placement.planner import CapacityExceededError, PlannerParameters, plan from sie_placement.report import IdentityStatus, Measurement, RequestReason, UnplacedReason, render from sie_placement.schema import ( BootEvidence, BootOutcome, + Capacity, Cell, ContainerShape, Demand, @@ -29,8 +30,10 @@ RequestShape, ServiceLevel, ServiceLevels, + SharedQuota, SourceRef, Throughput, + TypeCeiling, Warmth, ) @@ -169,11 +172,15 @@ def demand(hourly=None) -> Demand: ) -def run(service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, **overrides): +def run( + service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, capacity=None, **overrides +): parameters = PlannerParameters( **({"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides) ) - return plan(service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters) + return plan( + service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters, capacity=capacity + ) def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: @@ -644,6 +651,41 @@ def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(uns assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) +def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): + capacity = Capacity( + total=4, + types=(TypeCeiling(gpu_product=L4.gpu_product, accelerators=4),), + shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + ) + service_levels = levels(**{HOST: 0, GUEST: 60}) + result = run(service_levels, host_and_guest(), [load(GUEST)], capacity=capacity) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ + (1, 4, ("org-host-l4",)), + (1, 4, ("org-host-l4",)), + (3, 6, ("org-host-l4",)), + ] + with pytest.raises(CapacityExceededError) as caught: + run(service_levels, host_and_guest(guest_footprint=8000), [load(GUEST)], capacity=capacity) + assert [use.describe() for use in caught.value.exceeded] == [ + "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, above the total ceiling of 4", + "the container ceilings of org-guest-l4, org-host-l4 hold 8 NVIDIA-L4 accelerators, above the type ceiling of 4", + "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, " + "which with 2 held elsewhere exceed the shared quota of 6", + ] + + +def test_a_shared_lane_over_a_ceiling_is_refused_with_its_container_ceiling_intact(): + with pytest.raises(CapacityExceededError) as caught: + run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)], capacity=Capacity(total=3)) + assert ( + str(caught.value) == "the container ceilings of org-host-l4 hold 4 accelerators, above the total ceiling of 3" + ) + (refused,) = caught.value.result.plan.lanes + assert (refused.name, refused.max_containers) == ("org-host-l4", 4) + assert caught.value.result.plan == run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)]).plan + + def test_an_unshared_identity_keeps_a_current_shared_lane_it_already_holds(): current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current, unshared=frozenset({"org/a"})) From f653fb32f17445bff608868393ba229b95fa887c Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:14:59 +0100 Subject: [PATCH 18/40] fix(placement): refuse lanes and warmths the chart cannot realize - The Helm values emitter read only a lane's cell and dropped its fallback cells, so an ordered GPU candidate list emitted as a worker group on its first cell. A lane with fallback cells is now refused at its fallbackCells, since a worker group runs on one machine profile and GPU product. - An identity planned bridged, snapshot-cold or scheduled-warm is refused at its warmth: the overlay configures no remote bridge, and the chart restores no snapshot and keeps no warm window, so it would have run as a plain cold lane. - Tests pin that a lane pinning several identities, from a hand-written plan or from the planner's packing, carries every pinned identity on its own static queue pool and leaves lazy residents unpinned, that two such lanes on one GPU product emit through machine profiles and queue pools of their own, and that the default queue pool still refuses them. The chart render test checks the golden values keep each pinned lane on a pool and worker group of its own. --- packages/sie_placement/README.md | 25 +- .../sie_placement/src/sie_placement/helm.py | 31 +++ packages/sie_placement/tests/test_helm.py | 260 +++++++++++++++++- tools/ci/tests/test_helm_render.py | 10 + 4 files changed, 318 insertions(+), 8 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index cb6f5229b..3788370df 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -543,12 +543,22 @@ overlay sets: queue pool other than `default`, which the gateway needs before it admits that queue's workers. `gpus` lists every machine profile on the queue, `minimumWorkerCount` is the lanes' `floorMin`, `pinnedModels` are their - `pinned` identities, and `bundle` names the lanes' bundle when they share - one. + `pinned` identities, however many share a lane, and `bundle` names the + lanes' bundle when they share one. A `lazy-resident` identity is not pinned: + the workers of its lane load it on demand. It refuses, with an `UnrealizablePlanError` that names the plan field, a plan the chart and the gateway would not realize as planned: +- A lane with `fallbackCells`. A worker group runs on one machine profile and + GPU product and has no ordered candidate list, so the chart would place the + lane on its `cell` only. +- An identity whose warmth is `bridged`, `snapshot-cold` or `scheduled-warm`. + The overlay realizes `pinned` through a static queue pool's `pinnedModels` + and the lane's floor, `lazy-resident` through a warm lane whose workers load + the identity on demand, and `cold-with-bound` through a zero-floor lane that + scales from zero. It configures no remote bridge, and the chart restores no + snapshot and keeps no warm window. - A lane token the chart rejects, or a machine profile or bundle that is not a lowercase DNS-1123 label, which the chart requires of worker group and bundle keys. @@ -589,10 +599,13 @@ a lane on a named queue pool through that pool, for example These rules bound which plans emit. A worker group carries one queue pool, a named queue pool carries one floor and one pinned set, and the `default` queue pool carries no pins, so a pinned lane needs a named queue pool and a machine -profile of its own. The planner places each identity on a lane of its own, all -on the queue pool its parameters name, `default` unless set. Its plan -therefore emits only when no two identities share a machine profile and -bundle, and it pins nothing, or its queue pool is named and it has one lane. +profile of its own, and two pinned lanes on one GPU product need two machine +profiles. Such a lane may pin several identities and host lazy residents +beside them. The planner places every lane on the queue pool its parameters +name, `default` unless set, and may place several identities on one lane. Its +plan therefore emits only when no two lanes share a machine profile and bundle, +no lane has fallback cells and no identity is `snapshot-cold`, and it pins +nothing, or its queue pool is named and it has one lane. ## Tests diff --git a/packages/sie_placement/src/sie_placement/helm.py b/packages/sie_placement/src/sie_placement/helm.py index f9477ae49..fa1bc8e6e 100644 --- a/packages/sie_placement/src/sie_placement/helm.py +++ b/packages/sie_placement/src/sie_placement/helm.py @@ -26,6 +26,12 @@ ("sglang-cu130", ("sglang-cu130",)), ) +UNREALIZED_WARMTHS = { + Warmth.BRIDGED: "the overlay configures no remote bridge to serve it while the lane boots", + Warmth.SNAPSHOT_COLD: "the chart restores no snapshot, so the lane would boot cold", + Warmth.SCHEDULED_WARM: "the chart keeps no warm window, so the lane would stay cold inside it", +} + _LANE_TOKEN = re.compile(r"[A-Za-z0-9_-]+") _IDENTITY_TOKEN = re.compile(r"[a-z0-9]([-a-z0-9]*[a-z0-9])?") _GO_SPACE = ( @@ -162,6 +168,29 @@ def _pins_by_lane(plan: Plan) -> dict[str, dict[str, int]]: return pins +def _require_one_cell(path: str, lane: PlanLane) -> None: + if not lane.fallback_cells: + return + listed = ", ".join(f"{cell.machine_profile!r} with {cell.gpu_product!r}" for cell in lane.fallback_cells) + raise UnrealizablePlanError( + f"{path}.fallbackCells", + f"lists {listed} after the lane's cell, {lane.cell.machine_profile!r} with {lane.cell.gpu_product!r}, but a " + "worker group of the sie-cluster chart runs on one machine profile and GPU product and has no ordered " + "candidate list, so the chart would place the lane on its cell only; plan the lane on one cell", + ) + + +def _require_realizable_warmths(plan: Plan) -> None: + for index, entry in enumerate(plan.identities): + reason = UNREALIZED_WARMTHS.get(entry.warmth) + if reason is not None: + raise UnrealizablePlanError( + f"$.identities[{index}].warmth", + f"is {entry.warmth.value!r} for {entry.identity!r} on lane {entry.lane!r}, which the sie-cluster " + f"chart does not realize: {reason}; plan the identity pinned, lazy-resident or cold-with-bound", + ) + + def _require_realizable_floor(path: str, lane: PlanLane, parameters: ChartParameters) -> None: envelope = lane.envelope if envelope.floor_min or not envelope.floor_max: @@ -180,6 +209,7 @@ def _chart_lanes(plan: Plan, parameters: ChartParameters) -> list[_ChartLane]: lanes: dict[tuple[str, str, str], _ChartLane] = {} for index, lane in enumerate(plan.lanes): path = f"$.lanes[{index}]" + _require_one_cell(path, lane) key = ( lane_token(f"{path}.queuePool", lane.queue_pool), lane_token(f"{path}.cell.machineProfile", lane.cell.machine_profile), @@ -337,6 +367,7 @@ def _require_no_default_pool_pins(lanes: Iterable[_ChartLane]) -> None: def helm_values(plan: Plan, parameters: ChartParameters | None = None) -> dict[str, object]: parameters = ChartParameters() if parameters is None else parameters lanes = _chart_lanes(plan, parameters) + _require_realizable_warmths(plan) base = _enabled_base_bundles(parameters.base_pools) _require_lane_limit(lanes, base, parameters) by_profile: dict[str, list[_ChartLane]] = defaultdict(list) diff --git a/packages/sie_placement/tests/test_helm.py b/packages/sie_placement/tests/test_helm.py index b0ea6dd32..b5b230f5f 100644 --- a/packages/sie_placement/tests/test_helm.py +++ b/packages/sie_placement/tests/test_helm.py @@ -3,6 +3,7 @@ import json import random from dataclasses import replace +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -11,6 +12,7 @@ from sie_placement.helm import ( BUNDLED_DEPENDENCIES_LANE_LIMIT, EXTERNAL_DEPENDENCIES_LANE_LIMIT, + UNREALIZED_WARMTHS, ChartParameters, UnrealizablePlanError, chart_pools, @@ -21,17 +23,27 @@ from sie_placement.schema import ( WARM_FLOOR_WARMTHS, BootEvidence, + BootOutcome, Cell, ContainerShape, Demand, Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, Plan, PlanIdentity, PlanLane, PriceList, Receipt, + RequestShape, SchemaError, + ServiceLevel, ServiceLevels, + SourceRef, + Throughput, Warmth, ) @@ -248,8 +260,10 @@ def test_zero_floor_lane_accepts_a_realizable_floor( } -def test_schema_fixture_plan_has_an_unrealizable_zero_floor() -> None: - error = refusal(Plan.from_dict(load(FIXTURES / "plan.json"))) +def test_schema_fixture_plan_has_fallback_cells_and_an_unrealizable_zero_floor() -> None: + plan = Plan.from_dict(load(FIXTURES / "plan.json")) + assert refusal(plan).path == "$.lanes[1].fallbackCells" + error = refusal(replace(plan, lanes=tuple(replace(entry, fallback_cells=()) for entry in plan.lanes))) assert error.path == "$.lanes[1].envelope.floorMax" assert error.message.startswith("must be 0 or at least 2:") @@ -565,3 +579,245 @@ def test_every_warm_floor_warmth_is_carried_by_bundle_min_replicas() -> None: assert values["queueRouting"]["staticQueuePools"]["placement"]["minimumWorkerCount"] == 3 pinned = ["org/a"] if warmth is Warmth.PINNED else [] assert values["queueRouting"]["staticQueuePools"]["placement"]["pinnedModels"] == pinned + + +def residents(*lanes: PlanLane, placed: dict[str, tuple[str, Warmth]]) -> Plan: + identities = tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in placed.items() + ) + return Plan(placement="us-central1", lanes=lanes, identities=identities) + + +def static_pools(values: dict) -> dict: + return values["queueRouting"]["staticQueuePools"] + + +START = datetime(2026, 11, 2, tzinfo=UTC) +NO_DEMAND = Demand(start=START, end=START + timedelta(hours=24), series=()) +PACKING_PRICES = PriceList( + currency="USD", + machine_profiles=( + MachineProfilePrice( + name="l4", + gpu_product="NVIDIA-L4", + gpu_memory_mib=23034, + accelerator_per_hour=1.0, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + ), + MachineProfilePrice( + name="a100-80gb", + gpu_product="NVIDIA-A100-SXM4-80GB", + gpu_memory_mib=81920, + accelerator_per_hour=3.0, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + ), + ), + placements=(Placement(name="us-central1", region_multiplier=1),), +) +PACKING_PARAMETERS = planner.PlannerParameters( + placement="us-central1", max_containers=4, fallback_floor=2, cooldown_s=900, queue_pool="placement" +) + + +def packing_receipt( + identity: str, cell: Cell = L4, *, beside: tuple[str, ...] = (), footprint: int = 5000, working: int = 1000 +) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=SHAPE, + concurrency=8, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=8.0), + latency=Percentiles(p99_ms=150.0), + co_residents=beside, + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{'+'.join(beside)}"), + ) + + +def never_cold(identity: str, bound: int = 0) -> IdentityServiceLevel: + return IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + + +def packed_estate(queue_pool: str) -> planner.PlannerResult: + receipts = [ + packing_receipt("org/a", footprint=6000), + packing_receipt("org/b"), + packing_receipt("org/a", beside=("org/b",), footprint=6000, working=1500), + packing_receipt("org/b", beside=("org/a",), working=1500), + packing_receipt("org/c", footprint=4000, working=1500), + ] + load = BootEvidence( + identity="org/c", + cell=L4, + container_shape=SHAPE, + warmth=Warmth.LAZY_RESIDENT, + outcome=BootOutcome.SUCCEEDED, + boot_seconds=20.0, + source=SourceRef(uri="loads/org/c/l4"), + ) + return planner.plan( + ServiceLevels(identities=(never_cold("org/a"), never_cold("org/b"), never_cold("org/c", 60))), + receipts, + [load], + PACKING_PRICES, + NO_DEMAND, + None, + replace(PACKING_PARAMETERS, queue_pool=queue_pool), + ) + + +def scarce_type_attempts(cell: Cell) -> list[BootEvidence]: + return [ + BootEvidence( + identity="org/other", + cell=cell, + container_shape=SHAPE, + warmth=Warmth.COLD_WITH_BOUND, + outcome=BootOutcome.SUCCEEDED if index < 2 else BootOutcome.UNPLACED, + boot_seconds=30.0 if index < 2 else None, + source=SourceRef(uri=f"boots/org/other/{cell.machine_profile}/{index}"), + ) + for index in range(3) + ] + + +def test_a_lane_with_fallback_cells_is_refused() -> None: + error = refusal(plan_of(replace(lane("a"), fallback_cells=(A100,)))) + assert error.path == "$.lanes[0].fallbackCells" + assert error.message.startswith( + "lists 'a100-80gb' with 'NVIDIA-A100-SXM4-80GB' after the lane's cell, 'l4' with 'NVIDIA-L4', but a worker " + "group of the sie-cluster chart runs on one machine profile and GPU product and has no ordered candidate list" + ) + assert error.message.endswith("so the chart would place the lane on its cell only; plan the lane on one cell") + later = plan_of( + lane("a", floor_min=0), replace(lane("b", queue_pool="default", floor_min=0), fallback_cells=(A100,)) + ) + assert refusal(later).path == "$.lanes[1].fallbackCells" + + +def test_a_planner_candidate_list_is_refused_rather_than_emitted_on_its_first_cell() -> None: + receipts = [packing_receipt("org/model"), packing_receipt("org/model", A100)] + boots = [*scarce_type_attempts(L4), *scarce_type_attempts(A100)] + result = planner.plan( + ServiceLevels(identities=(never_cold("org/model"),)), + receipts, + boots, + PACKING_PRICES, + NO_DEMAND, + None, + replace(PACKING_PARAMETERS, hold_budget_s=20), + ) + (planned,) = result.plan.lanes + assert (planned.cell, planned.fallback_cells) == (L4, (A100,)) + error = refusal(result.plan) + assert error.path == "$.lanes[0].fallbackCells" + + +@pytest.mark.parametrize("warmth", sorted(UNREALIZED_WARMTHS)) +def test_a_warmth_the_chart_does_not_realize_is_refused(warmth: Warmth) -> None: + plan = plan_of(lane("a", floor_min=0), lane("b", queue_pool="default", floor_min=0), warmths={"b": warmth}) + error = refusal(plan) + assert error.path == "$.identities[1].warmth" + assert error.message.startswith( + f"is {warmth.value!r} for 'org/b' on lane 'b', which the sie-cluster chart does not realize: " + f"{UNREALIZED_WARMTHS[warmth]};" + ) + assert set(UNREALIZED_WARMTHS) == {Warmth.BRIDGED, Warmth.SNAPSHOT_COLD, Warmth.SCHEDULED_WARM} + + +def test_a_lane_pinning_several_identities_carries_them_all_on_its_own_static_queue_pool() -> None: + plan = residents( + lane("shared", floor_max=2), + lane("cold", queue_pool="default", cell=A100, floor_min=0), + placed={ + "org/b": ("shared", Warmth.PINNED), + "org/a": ("shared", Warmth.PINNED), + "org/c": ("shared", Warmth.LAZY_RESIDENT), + "org/d": ("cold", Warmth.COLD_WITH_BOUND), + }, + ) + values = helm_values(plan) + assert static_pools(values) == { + "placement": { + "bundle": "default", + "gpus": {"l4": 0}, + "minimumWorkerCount": 1, + "pinnedModels": ["org/a", "org/b"], + } + } + assert values["workers"]["pools"]["l4"]["bundles"] == { + "default": {"enabled": True, "minReplicas": 1, "maxReplicas": 4} + } + assert values["workers"]["pools"]["a100-80gb"]["queuePool"] == "default" + + +def co_pinned(name: str, profile: str, queue_pool: str) -> PlanLane: + return lane(name, queue_pool=queue_pool, cell=Cell(machine_profile=profile, gpu_product="NVIDIA-L4")) + + +def test_two_co_pinned_lanes_on_one_gpu_type_emit_through_their_own_machine_profiles_and_queue_pools() -> None: + placed = { + "org/a": ("first", Warmth.PINNED), + "org/b": ("first", Warmth.PINNED), + "org/c": ("second", Warmth.PINNED), + "org/d": ("second", Warmth.PINNED), + "org/e": ("second", Warmth.LAZY_RESIDENT), + } + values = helm_values( + residents(co_pinned("first", "l4", "first"), co_pinned("second", "l4-b", "second"), placed=placed) + ) + assert {profile: group["queuePool"] for profile, group in values["workers"]["pools"].items()} == { + "l4": "first", + "l4-b": "second", + } + assert {name: pool["pinnedModels"] for name, pool in static_pools(values).items()} == { + "first": ["org/a", "org/b"], + "second": ["org/c", "org/d"], + } + error = refusal(residents(co_pinned("first", "l4", "first"), co_pinned("second", "l4", "second"), placed=placed)) + assert error.path == "$.lanes[1].queuePool" + assert error.message.startswith("shares worker group 'l4' with $.lanes[0], which has 'first', got 'second';") + shared_pool = refusal( + residents(co_pinned("first", "l4", "shared"), co_pinned("second", "l4-b", "shared"), placed=placed) + ) + assert shared_pool.path == "$.lanes[1].name" + assert "which pins ['org/a', 'org/b'], but pins ['org/c', 'org/d']" in shared_pool.message + + +def test_a_lane_pinning_several_identities_on_the_default_queue_pool_is_refused() -> None: + placed = {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)} + error = refusal(residents(lane("shared", queue_pool="default"), placed=placed)) + assert error.path == "$.identities[0].warmth" + assert "pins 'org/a' on chart lane 'default/l4/default'" in error.message + + +def test_a_planner_lane_pinning_several_identities_emits_on_a_named_queue_pool() -> None: + result = packed_estate("placement") + (planned,) = result.plan.lanes + assert {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} == { + "org/a": (planned.name, Warmth.PINNED), + "org/b": (planned.name, Warmth.PINNED), + "org/c": (planned.name, Warmth.LAZY_RESIDENT), + } + values = helm_values(result.plan) + assert static_pools(values) == { + "placement": { + "bundle": "default", + "gpus": {"l4": 0}, + "minimumWorkerCount": planned.envelope.floor_min, + "pinnedModels": ["org/a", "org/b"], + } + } + assert values["workers"]["pools"]["l4"]["bundles"]["default"] == { + "enabled": True, + "minReplicas": planned.envelope.floor_min, + "maxReplicas": planned.max_containers, + } + error = refusal(packed_estate("default").plan) + assert error.path.endswith(".warmth") + assert "place pinned identities on a named queuePool" in error.message diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 15db7f278..4d2313964 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4142,6 +4142,16 @@ def test_placement_values_render_the_planned_lanes(tmp_path: Path) -> None: gateway = container_env(docs, "sie-sie-cluster-gateway", "gateway") static_queue_pools = overlay["queueRouting"]["staticQueuePools"] assert json.loads(gateway["SIE_GATEWAY_STATIC_QUEUE_POOLS"]["value"]) == static_queue_pools + pinned = {name: pool for name, pool in static_queue_pools.items() if pool["pinnedModels"]} + assert any(len(pool["pinnedModels"]) > 1 for pool in pinned.values()) + products = [] + for name, pool in pinned.items(): + (profile,) = pool["gpus"] + group = overlay["workers"]["pools"][profile] + assert group["queuePool"] == name + assert [lane for lane in lanes if lane[0] == name] == [(name, profile, pool["bundle"])] + products.append(group["gpu"]["product"]) + assert len(set(products)) < len(products) scaled_objects = { tuple(scaled_object["metadata"]["labels"][f"sie.superlinked.com/{label}"] for label in LANE_LABELS): ( From 85313d89c1e0c95d23eca3b46cde0b411544d482 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:30:26 +0100 Subject: [PATCH 19/40] fix(placement): put each pinning lane on a named queue pool of its own The sie-cluster chart carries a lane's pinned set on a static queue pool, and the default queue pool carries no pins, so a plan with more than one pinned lane on one queue pool could not be emitted as Helm values. The planner now gives every lane that pins an identity a named queue pool derived from the lane name, unique within the plan, keeps a named pool the current plan already gives the lane, and leaves lazy-only and cold lanes on the parameters' queue pool. A lane's queue pool no longer decides whether a current placement, a lazy host or a packing target is reusable, since the pool follows from the lane. --- packages/sie_placement/README.md | 21 ++- .../src/sie_placement/planner.py | 39 ++++- .../tests/fixtures/planner/expected-plan.json | 4 +- .../fixtures/planner/expected-report.json | 58 ++++++-- .../sie_placement/tests/test_availability.py | 6 +- packages/sie_placement/tests/test_levers.py | 2 +- packages/sie_placement/tests/test_packing.py | 6 +- packages/sie_placement/tests/test_planner.py | 139 ++++++++++++++++-- 8 files changed, 229 insertions(+), 46 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 28fcd22cc..ec374571d 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -99,6 +99,16 @@ refers to its lane by that name. Two plan lanes may carry the same three tokens, and so name one chart lane, which cannot give them separate envelopes or concurrency; the planner's report lists every such chart lane. +The planner gives each lane that pins at least one identity a named queue pool +of its own, since the chart carries a lane's pinned set on a static queue pool +and the default queue pool carries no pins. A lane the current plan already +puts on a named queue pool keeps it, so planning again leaves the pool as it +is. Otherwise the pool is derived from the lane's `name`: the name itself, cut +to 63 characters with a digest of the name appended when it is longer or when +another lane of the plan, `default`, or the parameters' `queue_pool` already +has it. A lane on which every identity is lazy-resident or cold is on the +parameters' `queue_pool`. Identities that share a lane share its queue pool. + `fallbackCells` is an optional ordered candidate list: cells, after `cell`, that the lane's containers may be placed on when `cell` cannot be placed. No fallback cell repeats `cell` or another fallback cell. A realized card is @@ -220,7 +230,7 @@ choice, such as its own previous plan, reports no change. | `buffer_containers` | The declared warm buffer above a lane's floor, `0` by default | | `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | -| `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | +| `queue_pool`, `bundle`, `bundles` | The queue pool of every lane that pins no identity, and each identity's bundle: its `bundles` entry, otherwise `bundle` | | `unshared` | Identities that never share a lane: none is packed or made a lazy resident beside another identity, and no other identity joins a lane one of them is on. Empty by default | ### Selection @@ -256,8 +266,7 @@ document: `comeAliveP95S` of `0` admits `pinned` only. 4. A cell also offers `lazy-resident` on each host lane with its cell, container shape and concurrency. A host lane is a lane of the current plan - with a `floorMin` of at least 1, on the parameters' queue pool and the - identity's bundle, on which another identity whose `comeAliveP95S` is `0` + with a `floorMin` of at least 1, on the identity's bundle, on which another identity whose `comeAliveP95S` is `0` stays pinned in this run, on its current placement or kept there. Those identities are placed first: a bound of `0` admits no cold or lazy option, so no other choice moves them. The identity's own current lane is also a @@ -288,8 +297,8 @@ document: The current placement is the identity's current lane when that lane has the option's cell and fallback cells, container shape and concurrency, the -identity's warmth is the option's, and the lane's `queuePool` and `bundle` are -the ones the parameters give the identity; a `lazy-resident` option must also be +identity's warmth is the option's, and the lane's `bundle` is the one the +parameters give the identity; a `lazy-resident` option must also be hosted by that lane, and a `pinned` option on a lane where other identities are pinned must stand on a co-residency receipt beside them. A current placement that no option matches lacks evidence: a receipt at its lane's cell, container @@ -337,7 +346,7 @@ plan total does not fall. 4. Packing. The planner visits every placed identity that is alone on its lane and is neither a generation nor an `unshared` identity, in decreasing order of its lane's floor cost, ties broken by identity. For each it tries the - other lanes of the plan on its queue pool and bundle that keep a floor and + other lanes of the plan on its bundle that keep a floor and host a pinned identity, where it has a receipt at the lane's cell, container shape and concurrency: first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, since nothing moves that identity, then in diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index b24c2e805..6db08526f 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -67,6 +67,8 @@ PLACEMENT_FAILURE_FIELD = "placementFailureRate" MEMORY_HEADROOM_FRACTION = 0.8 COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) +DEFAULT_QUEUE_POOL = "default" +QUEUE_POOL_MAX_LENGTH = 63 _SLUG = re.compile(r"[^a-z0-9]+") _LEVEL_FIELDS = { @@ -100,7 +102,7 @@ class PlannerParameters: buffer_containers: int = 0 hold_budget_s: int = 0 hours_per_month: float = HOURS_PER_MONTH - queue_pool: str = "default" + queue_pool: str = DEFAULT_QUEUE_POOL bundle: str = "default" bundles: Mapping[str, str] = field(default_factory=dict) unshared: frozenset[str] = frozenset() @@ -383,6 +385,18 @@ def _digest(value: str, width: int = 8) -> str: return hashlib.sha256(value.encode()).hexdigest()[:width] +def _named_queue_pool(lane: str, taken: set[str]) -> str: + base = _slug(lane) or "lane" + candidates = [base] if len(base) <= QUEUE_POOL_MAX_LENGTH else [] + candidates.extend( + f"{base[: QUEUE_POOL_MAX_LENGTH - width - 1].rstrip('-')}-{_digest(lane, width)}" for width in (8, 16) + ) + name = next((candidate for candidate in candidates if candidate not in taken), None) + if name is None: + raise PlannerInputError(f"queue pool name for lane {lane!r} is already taken") + return name + + def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> str | None: price = prices.get(cell.machine_profile) if price is None: @@ -667,7 +681,6 @@ def is_current( return False return ( entry.warmth is warmth - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and lane.cells == (receipt.cell, *fallbacks) and _runs_on(receipt, lane) @@ -798,7 +811,7 @@ def keepable(self, identity: str) -> tuple[PlanIdentity, PlanLane] | None: if current is None: return None lane = current[1] - if lane.queue_pool != self.parameters.queue_pool or lane.bundle != self.parameters.bundle_of(identity): + if lane.bundle != self.parameters.bundle_of(identity): return None return current @@ -828,7 +841,6 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and not self.hosts_solitary((*self.current_pinned[name], *self.current_lazy[name])) - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -1461,6 +1473,21 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: envelope=envelope_for(option.warmth, self.parameters), ) + def queue_pools(self, lanes: Iterable[str], identities: Iterable[PlanIdentity]) -> dict[str, str]: + pools = dict.fromkeys(lanes, self.parameters.queue_pool) + pinning = sorted({entry.lane for entry in identities if entry.warmth is Warmth.PINNED}) + taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool} + for name in pinning: + current = self.current_lanes.get(name) + if current is not None and current.queue_pool not in taken: + pools[name] = current.queue_pool + taken.add(current.queue_pool) + for name in pinning: + if pools[name] == self.parameters.queue_pool: + pools[name] = _named_queue_pool(name, taken) + taken.add(pools[name]) + return pools + def within_ceiling(self, lane: PlanLane) -> PlanLane: ceiling = self.parameters.max_containers if lane.max_containers <= ceiling: @@ -1545,9 +1572,10 @@ def assemble(self, choices: Mapping[str, _Choice]) -> _Layout: PlanIdentity(identity=identity, lane=names[identity], warmth=option.warmth) for identity, option in placed.items() ) + pools = self.queue_pools(lanes, identities) plan = Plan( placement=self.parameters.placement, - lanes=tuple(lanes[name] for name in sorted(lanes)), + lanes=tuple(replace(lanes[name], queue_pool=pools[name]) for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} @@ -1639,7 +1667,6 @@ def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, f and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and lane.container_shape.gpu_count == 1 - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and pinned[lane.name] and not self.hosts_solitary((*pinned[lane.name], *lazy[lane.name])) diff --git a/packages/sie_placement/tests/fixtures/planner/expected-plan.json b/packages/sie_placement/tests/fixtures/planner/expected-plan.json index 303a4ed79..32c31b3be 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-plan.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-plan.json @@ -25,7 +25,7 @@ }, { "name": "default-l4-default", - "queuePool": "default", + "queuePool": "default-l4-default", "bundle": "default", "cell": { "machineProfile": "l4", @@ -65,7 +65,7 @@ }, { "name": "qwen-qwen3-embedding-4b-l4", - "queuePool": "default", + "queuePool": "qwen-qwen3-embedding-4b-l4", "bundle": "default", "cell": { "machineProfile": "l4", diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 226647ae1..44b50fb0c 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -327,18 +327,7 @@ } ], "capacity": [], - "sharedChartLanes": [ - { - "queuePool": "default", - "machineProfile": "l4", - "bundle": "default", - "lanes": [ - "baai-bge-m3-sparse-l4", - "default-l4-default", - "qwen-qwen3-embedding-4b-l4" - ] - } - ], + "sharedChartLanes": [], "measurementRequests": [ { "identity": "Qwen/Qwen3-Reranker-0.6B", @@ -632,11 +621,54 @@ } } }, + { + "name": "default-l4-default", + "before": { + "name": "default-l4-default", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + "after": { + "name": "default-l4-default", + "queuePool": "default-l4-default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + } + }, { "name": "qwen-qwen3-embedding-4b-l4", "after": { "name": "qwen-qwen3-embedding-4b-l4", - "queuePool": "default", + "queuePool": "qwen-qwen3-embedding-4b-l4", "bundle": "default", "cell": { "machineProfile": "l4", diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index 98a0a683d..b1c21433b 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -191,9 +191,13 @@ def lane(name, cell=L4, *, fallbacks=(), floor=1, buffer=0) -> PlanLane: def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index aec54c2bf..8721c77a5 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -125,7 +125,7 @@ def shared_lane(*identities): lanes=( PlanLane( name="shared", - queue_pool="default", + queue_pool="shared", bundle="default", cell=L4, container_shape=SHAPE, diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index b3287c54f..ecc2f36d5 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -197,9 +197,13 @@ def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index e0d78d211..f5a73fcb3 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -239,7 +239,7 @@ def single_lane_plan( lanes=( PlanLane( name=name, - queue_pool="default", + queue_pool=name if warmth is Warmth.PINNED else "default", bundle="default", cell=cell, container_shape=SHAPE, @@ -271,9 +271,13 @@ def plan_lane(name, *, cell=L4, floor=1, bundle="default", queue_pool="default") def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) @@ -1331,9 +1335,11 @@ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_th assert result.report.monthly_cost == pytest.approx(MONTH) -@pytest.mark.parametrize("shared", [plan_lane("shared", bundle="sglang"), plan_lane("shared", queue_pool="batch")]) -def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): - current = plan_of([plan_lane("own"), shared], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}) +def test_a_lazy_resident_needs_a_lane_of_its_bundle(): + current = plan_of( + [plan_lane("own"), plan_lane("shared", bundle="sglang")], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) assert {entry.identity: entry.lane for entry in result.plan.identities} == { ANCHOR: "org-anchor-l4", @@ -1343,6 +1349,20 @@ def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): assert all(request.lane != "shared" for request in result.report.measurement_requests) +def test_a_lazy_resident_joins_a_pinned_lane_on_its_named_queue_pool(): + current = plan_of( + [plan_lane("own"), plan_lane("shared", queue_pool="batch")], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + hourly = {IDENTITY: {3: 90000}} + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, hourly=hourly, current=current) + assert [(entry.identity, entry.lane, entry.warmth) for entry in result.plan.identities] == [ + (ANCHOR, "shared", Warmth.PINNED), + (IDENTITY, "shared", Warmth.LAZY_RESIDENT), + ] + assert result.plan.lanes == (current.lanes[1],) + + def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_current_lane_and_on_a_packed_one(): current = plan_of( [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} @@ -1427,17 +1447,25 @@ def test_a_placement_that_moves_leaves_its_shared_lane_to_the_kept_neighbour(): @pytest.mark.parametrize( - ("overrides", "moved"), - [({"bundles": {IDENTITY: "sglang"}}, ("default", "sglang")), ({"queue_pool": "batch"}, ("batch", "default"))], + ("overrides", "expected"), + [ + ( + {"bundles": {IDENTITY: "sglang"}}, + {"org-model-l4": ("org-model-l4", "sglang"), "shared": ("shared", "default")}, + ), + ({"queue_pool": "batch"}, {"shared": ("shared", "default")}), + ], ) -def test_a_new_queue_pool_or_bundle_moves_the_identity_off_its_current_lane(overrides, moved): +def test_a_new_bundle_moves_the_identity_off_its_current_lane_and_a_new_default_queue_pool_does_not( + overrides, expected +): current = single_lane_plan(others=("org/kept",), max_containers=4) service_levels = levels( (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) ) result = run(service_levels, [receipt()], prices=L4_ONLY, current=current, **overrides) lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} - assert lanes == {"org-model-l4": moved, "shared": ("default", "default")} + assert lanes == expected def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): @@ -1455,7 +1483,9 @@ def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): def test_plan_lanes_that_name_one_chart_lane_are_reported(): service_levels = levels(("org/a", WARM), ("org/b", WARM), ("org/c", WARM)) receipts = [receipt(identity=identity) for identity in ("org/a", "org/b", "org/c")] - result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/c": "sglang"}) + boots = [boot(identity=identity) for identity in ("org/a", "org/b", "org/c")] + result = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/c": "sglang"}) + assert {entry.warmth for entry in result.plan.identities} == {Warmth.COLD_WITH_BOUND} (shared,) = result.report.shared_chart_lanes assert shared.to_dict() == { "queuePool": "default", @@ -1463,8 +1493,77 @@ def test_plan_lanes_that_name_one_chart_lane_are_reported(): "bundle": "default", "lanes": ["org-a-l4", "org-b-l4"], } - apart = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "a", "org/b": "b"}) + apart = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/a": "a", "org/b": "b"}) assert apart.report.shared_chart_lanes == () + pinned = run(service_levels, receipts, prices=L4_ONLY) + assert {entry.warmth for entry in pinned.plan.identities} == {Warmth.PINNED} + assert pinned.report.shared_chart_lanes == () + + +def test_each_lane_that_pins_an_identity_gets_its_own_named_queue_pool(): + long = "org/" + "x" * 70 + never_cold = ServiceLevel(come_alive_p95_s=0, warm_p99_ms=ANY_P99_MS) + service_levels = levels(("org/a", never_cold), ("org/b", never_cold), (long, never_cold), ("org/c", WARM)) + receipts = [ + receipt(identity="org/a"), + receipt(identity="org/b", cell=A100), + receipt(identity=long, cell=A100), + receipt(identity="org/c"), + ] + result = run(service_levels, receipts, [boot(identity="org/c")]) + warmths = {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} + long_lane = warmths[long][0] + assert warmths == { + "org/a": ("org-a-l4", Warmth.PINNED), + "org/b": ("org-b-a100-80gb", Warmth.PINNED), + long: (long_lane, Warmth.PINNED), + "org/c": ("org-c-l4", Warmth.COLD_WITH_BOUND), + } + pools = {lane.name: lane.queue_pool for lane in result.plan.lanes} + long_pool = f"{long_lane[:54]}-{hashlib.sha256(long_lane.encode()).hexdigest()[:8]}" + assert pools == { + "org-a-l4": "org-a-l4", + "org-b-a100-80gb": "org-b-a100-80gb", + long_lane: long_pool, + "org-c-l4": "default", + } + assert len(long_pool) == 63 + again = run(service_levels, receipts, [boot(identity="org/c")], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_the_default_pool(): + named = single_lane_plan(name="kept", identity="org/kept") + named_lane = replace(named.lanes[0], queue_pool="org-model-l4") + unnamed = single_lane_plan(name="org-model-l4") + unnamed_lane = replace(unnamed.lanes[0], queue_pool="default") + current = Plan( + placement="test", + lanes=(unnamed_lane, named_lane), + identities=(*unnamed.identities, *named.identities), + ) + service_levels = levels( + (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) + ) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities} == { + IDENTITY: "org-model-l4", + "org/kept": "kept", + } + digest = hashlib.sha256(b"org-model-l4").hexdigest()[:8] + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == { + "kept": "org-model-l4", + "org-model-l4": f"org-model-l4-{digest}", + } + assert [(change.name, change.after.queue_pool) for change in result.report.lane_changes] == [ + ("org-model-l4", f"org-model-l4-{digest}") + ] + moved = run(service_levels, [receipt()], prices=L4_ONLY, current=current, queue_pool="org-model-l4") + assert {lane.name: lane.queue_pool for lane in moved.plan.lanes} == { + "kept": "kept", + "org-model-l4": f"org-model-l4-{digest}", + } def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): @@ -1489,18 +1588,19 @@ def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): assert {lane.name: lane for lane in result.plan.lanes}[f"org-model-l4-{digest[:8]}"] == second.lanes[0] -def test_bundles_and_the_queue_pool_come_from_the_parameters(): +def test_bundles_and_the_default_queue_pool_come_from_the_parameters(): service_levels = levels(("org/embed", WARM), ("org/generate", WARM)) result = run( service_levels, [receipt(identity="org/embed"), receipt(identity="org/generate")], + [boot(identity="org/embed")], prices=L4_ONLY, queue_pool="batch", bundle="default", bundles={"org/generate": "sglang"}, ) lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} - assert lanes == {"org-embed-l4": ("batch", "default"), "org-generate-l4": ("batch", "sglang")} + assert lanes == {"org-embed-l4": ("batch", "default"), "org-generate-l4": ("org-generate-l4", "sglang")} def test_equal_costs_prefer_the_current_placement(): @@ -1684,8 +1784,13 @@ def test_the_estate_report_states_its_findings(): assert shared.monthly_cost == pytest.approx(dense.monthly_cost) assert shared.unpriced_identities == ("intfloat/e5-base-v2",) assert report.unpriced_lanes == ("default-l4-default",) - (chart_lane,) = report.shared_chart_lanes - assert chart_lane.lanes == ("baai-bge-m3-sparse-l4", "default-l4-default", "qwen-qwen3-embedding-4b-l4") + assert report.shared_chart_lanes == () + assert {lane.name: lane.queue_pool for lane in plan(**estate()).plan.lanes} == { + "baai-bge-m3-sparse-l4": "default", + "default-l4-default": "default-l4-default", + "qwen-qwen3-8b-a100-80gb": "default", + "qwen-qwen3-embedding-4b-l4": "qwen-qwen3-embedding-4b-l4", + } current = estate()["current_plan"] assert sum(lane.envelope.floor_min for lane in plan(**estate()).plan.lanes) == 1 + sum( lane.envelope.floor_min for lane in current.lanes @@ -1942,9 +2047,11 @@ def test_random_current_estates_keep_what_no_measured_option_beats(seed): assert Plan.from_dict(result.plan.to_dict()) == result.plan before = {lane.name: lane for lane in current.lanes} after = {lane.name: lane for lane in result.plan.lanes} + pinning = {entry.lane for entry in result.plan.identities if entry.warmth is Warmth.PINNED} for name, lane in after.items(): if name in before: - assert lane == capped(before[name], 4) + assert replace(lane, queue_pool=before[name].queue_pool) == capped(before[name], 4) + assert lane.queue_pool == (name if name in pinning else "default") placements = {entry.identity: entry for entry in result.plan.identities} never_cold = {entry.identity for entry in service_levels.identities if entry.level.come_alive_p95_s == 0} anchored = { From 33fafeb155dfca124a5e3014cc55bcc5443de903 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:34:52 +0100 Subject: [PATCH 20/40] test(placement): emit a planner estate with several pinned lanes as Helm values The planner now gives each pinning lane a named queue pool of its own, so a plan that pins identities on several lanes, each on its own machine profile, emits one static queue pool per pinned lane and keeps its lazy-only and cold lanes on the default pool. The estate fixture now fails on two queue pools sharing the l4 worker group rather than on one chart lane named twice. --- packages/sie_placement/README.md | 11 +-- packages/sie_placement/tests/test_helm.py | 85 ++++++++++++++++++----- 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 86aa8a4f5..433d37f74 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -610,11 +610,12 @@ named queue pool carries one floor and one pinned set, and the `default` queue pool carries no pins, so a pinned lane needs a named queue pool and a machine profile of its own, and two pinned lanes on one GPU product need two machine profiles. Such a lane may pin several identities and host lazy residents -beside them. The planner places every lane on the queue pool its parameters -name, `default` unless set, and may place several identities on one lane. Its -plan therefore emits only when no two lanes share a machine profile and bundle, -no lane has fallback cells and no identity is `snapshot-cold`, and it pins -nothing, or its queue pool is named and it has one lane. +beside them. The planner gives every lane that pins an identity a named queue +pool of its own, places every other lane on the queue pool its parameters name, +`default` unless set, and may place several identities on one lane. Its plan +therefore emits only when no two lanes share a machine profile and bundle, no +lane that pins an identity shares its machine profile with another lane, no lane +has fallback cells and no identity is `snapshot-cold`. ## Tests diff --git a/packages/sie_placement/tests/test_helm.py b/packages/sie_placement/tests/test_helm.py index b5b230f5f..e45c0b9d5 100644 --- a/packages/sie_placement/tests/test_helm.py +++ b/packages/sie_placement/tests/test_helm.py @@ -268,10 +268,12 @@ def test_schema_fixture_plan_has_fallback_cells_and_an_unrealizable_zero_floor() assert error.message.startswith("must be 0 or at least 2:") -def test_planner_estate_plan_names_one_chart_lane_twice() -> None: +def test_planner_estate_plan_puts_two_queue_pools_on_one_worker_group() -> None: error = refusal(Plan.from_dict(load(FIXTURES / "planner" / "expected-plan.json"))) - assert error.path == "$.lanes[1]" - assert error.message.startswith("names chart lane 'default/l4/default', which $.lanes[0] also names;") + assert error.path == "$.lanes[1].queuePool" + assert error.message.startswith( + "shares worker group 'l4' with $.lanes[0], which has 'default', got 'default-l4-default';" + ) def planner_estate(identities: set[str], parameters: planner.PlannerParameters) -> planner.PlannerResult: @@ -307,19 +309,17 @@ def test_a_planner_plan_with_one_lane_per_chart_lane_emits_its_lanes() -> None: assert "queueRouting" not in values -@pytest.mark.parametrize( - ("queue_pool", "path", "message"), - [ - ("default", "$.identities[0].warmth", "pins 'BAAI/bge-m3' on chart lane 'default/l4/default', but the"), - ("placement", "$.lanes[1].envelope.floorMin", "shares static queue pool 'placement' with $.lanes[0], whose"), - ], -) -def test_a_planner_plan_that_pins_beside_another_lane_is_refused(queue_pool: str, path: str, message: str) -> None: +@pytest.mark.parametrize("queue_pool", ["default", "placement"]) +def test_a_planner_plan_that_pins_beside_another_lane_emits_the_pinned_lane_on_its_own_queue_pool( + queue_pool: str, +) -> None: result = planner_estate({"BAAI/bge-m3", "Qwen/Qwen3-8B"}, replace(PLANNER_PARAMETERS, queue_pool=queue_pool)) assert [entry.warmth for entry in result.plan.identities] == [Warmth.PINNED, Warmth.COLD_WITH_BOUND] - error = refusal(result.plan) - assert error.path == path - assert error.message.startswith(message) + pinned, cold = (entry.lane for entry in result.plan.identities) + assert {entry.name: entry.queue_pool for entry in result.plan.lanes} == {pinned: pinned, cold: queue_pool} + values = helm_values(result.plan, ChartParameters(fallback_replicas=PLANNER_PARAMETERS.fallback_floor)) + assert static_pools(values)[pinned]["pinnedModels"] == ["BAAI/bge-m3"] + assert set(static_pools(values)) == {pinned} | ({queue_pool} - {"default"}) def bundle_lanes(count: int) -> Plan: @@ -805,8 +805,9 @@ def test_a_planner_lane_pinning_several_identities_emits_on_a_named_queue_pool() "org/c": (planned.name, Warmth.LAZY_RESIDENT), } values = helm_values(result.plan) + assert planned.queue_pool == planned.name assert static_pools(values) == { - "placement": { + planned.name: { "bundle": "default", "gpus": {"l4": 0}, "minimumWorkerCount": planned.envelope.floor_min, @@ -818,6 +819,54 @@ def test_a_planner_lane_pinning_several_identities_emits_on_a_named_queue_pool() "minReplicas": planned.envelope.floor_min, "maxReplicas": planned.max_containers, } - error = refusal(packed_estate("default").plan) - assert error.path.endswith(".warmth") - assert "place pinned identities on a named queuePool" in error.message + assert helm_values(packed_estate("default").plan) == values + + +def test_a_planner_estate_with_several_pinned_lanes_emits_a_named_queue_pool_for_each() -> None: + l4_b = Cell(machine_profile="l4-b", gpu_product="NVIDIA-L4") + second_l4 = replace(PACKING_PRICES.machine_profiles[0], name="l4-b") + prices = replace(PACKING_PRICES, machine_profiles=(*PACKING_PRICES.machine_profiles, second_l4)) + boot = BootEvidence( + identity="org/d", + cell=A100, + container_shape=SHAPE, + warmth=Warmth.COLD_WITH_BOUND, + outcome=BootOutcome.SUCCEEDED, + boot_seconds=60.0, + source=SourceRef(uri="boots/org/d/a100-80gb"), + ) + result = planner.plan( + ServiceLevels(identities=(never_cold("org/a"), never_cold("org/b"), never_cold("org/d", 600))), + [packing_receipt("org/a"), packing_receipt("org/b", l4_b), packing_receipt("org/d", A100)], + [boot], + prices, + NO_DEMAND, + None, + replace(PACKING_PARAMETERS, queue_pool="default"), + ) + assert {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} == { + "org/a": ("org-a-l4", Warmth.PINNED), + "org/b": ("org-b-l4-b", Warmth.PINNED), + "org/d": ("org-d-a100-80gb", Warmth.COLD_WITH_BOUND), + } + values = helm_values(result.plan, ChartParameters(fallback_replicas=PACKING_PARAMETERS.fallback_floor)) + assert {profile: group["queuePool"] for profile, group in values["workers"]["pools"].items()} == { + "a100-80gb": "default", + "l4": "org-a-l4", + "l4-b": "org-b-l4-b", + } + assert static_pools(values) == { + "org-a-l4": {"bundle": "default", "gpus": {"l4": 0}, "minimumWorkerCount": 1, "pinnedModels": ["org/a"]}, + "org-b-l4-b": {"bundle": "default", "gpus": {"l4-b": 0}, "minimumWorkerCount": 1, "pinnedModels": ["org/b"]}, + } + replanned = planner.plan( + ServiceLevels(identities=(never_cold("org/a"), never_cold("org/b"), never_cold("org/d", 600))), + [packing_receipt("org/a"), packing_receipt("org/b", l4_b), packing_receipt("org/d", A100)], + [boot], + prices, + NO_DEMAND, + result.plan, + replace(PACKING_PARAMETERS, queue_pool="default"), + ) + assert replanned.plan == result.plan + assert helm_values(replanned.plan, ChartParameters(fallback_replicas=PACKING_PARAMETERS.fallback_floor)) == values From 0c7c09c423c78f5a71d2fca77437eae2a4381ed3 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:41:32 +0100 Subject: [PATCH 21/40] refactor(placement): share the Helm emitter's plan refusals with the tune applier plan_refusals in sie_placement.helm lists the refusals of helm_values that hold whatever values lie beneath the overlay: a lane with fallback cells, a pin on the default queue pool and a warmth the chart does not realize. helm_values raises them where it did, and sie-placement tune reports every one of them in the emitter's words, so the applier now also refuses a lane with fallback cells. --- packages/sie_placement/README.md | 10 ++-- .../sie_placement/src/sie_placement/helm.py | 52 +++++++++++++------ .../sie_placement/src/sie_placement/keda.py | 35 ++----------- packages/sie_placement/tests/test_helm.py | 24 +++++++++ packages/sie_placement/tests/test_keda.py | 33 ++++++++++-- 5 files changed, 99 insertions(+), 55 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 71f18210b..4c6215ee0 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -340,10 +340,12 @@ The command exits with status 2 before it calls Prometheus or the gateway when: - a lane's `queuePool`, machine profile or bundle is not 1 to 63 letters, digits, `_` or `-`, two lanes need the same carrier, or a carrier name is longer than the gateway's 128 characters; -- the `sie-cluster` chart cannot run the plan: a lane on the `default` queue - pool pins an identity, whose pins no static queue pool carries, or an - identity is `bridged`, `snapshot-cold` or `scheduled-warm`, warmths the chart - does not realize; +- the `sie-cluster` chart cannot run the plan: a lane has fallback cells, a lane + on the `default` queue pool pins an identity, whose pins no static queue pool + carries, or an identity is `bridged`, `snapshot-cold` or `scheduled-warm`, + warmths the chart does not realize. These are `plan_refusals` of + `sie_placement.helm`, the refusals of `helm_values` that hold whatever values + lie beneath the overlay, reported in its words; - a token file or `SIE_ADMIN_TOKEN` holds anything but one run of printable ASCII characters without spaces after trimming, or the Prometheus URL has an invalid port. diff --git a/packages/sie_placement/src/sie_placement/helm.py b/packages/sie_placement/src/sie_placement/helm.py index fa1bc8e6e..91d234bb1 100644 --- a/packages/sie_placement/src/sie_placement/helm.py +++ b/packages/sie_placement/src/sie_placement/helm.py @@ -5,7 +5,7 @@ import math import re from collections import defaultdict -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field from sie_placement.schema import ContainerShape, Plan, PlanLane, SchemaError, Warmth @@ -168,11 +168,11 @@ def _pins_by_lane(plan: Plan) -> dict[str, dict[str, int]]: return pins -def _require_one_cell(path: str, lane: PlanLane) -> None: +def _one_cell_error(path: str, lane: PlanLane) -> UnrealizablePlanError | None: if not lane.fallback_cells: - return + return None listed = ", ".join(f"{cell.machine_profile!r} with {cell.gpu_product!r}" for cell in lane.fallback_cells) - raise UnrealizablePlanError( + return UnrealizablePlanError( f"{path}.fallbackCells", f"lists {listed} after the lane's cell, {lane.cell.machine_profile!r} with {lane.cell.gpu_product!r}, but a " "worker group of the sie-cluster chart runs on one machine profile and GPU product and has no ordered " @@ -180,11 +180,11 @@ def _require_one_cell(path: str, lane: PlanLane) -> None: ) -def _require_realizable_warmths(plan: Plan) -> None: +def _warmth_errors(plan: Plan) -> Iterator[UnrealizablePlanError]: for index, entry in enumerate(plan.identities): reason = UNREALIZED_WARMTHS.get(entry.warmth) if reason is not None: - raise UnrealizablePlanError( + yield UnrealizablePlanError( f"$.identities[{index}].warmth", f"is {entry.warmth.value!r} for {entry.identity!r} on lane {entry.lane!r}, which the sie-cluster " f"chart does not realize: {reason}; plan the identity pinned, lazy-resident or cold-with-bound", @@ -209,7 +209,9 @@ def _chart_lanes(plan: Plan, parameters: ChartParameters) -> list[_ChartLane]: lanes: dict[tuple[str, str, str], _ChartLane] = {} for index, lane in enumerate(plan.lanes): path = f"$.lanes[{index}]" - _require_one_cell(path, lane) + one_cell = _one_cell_error(path, lane) + if one_cell is not None: + raise one_cell key = ( lane_token(f"{path}.queuePool", lane.queue_pool), lane_token(f"{path}.cell.machineProfile", lane.cell.machine_profile), @@ -352,22 +354,42 @@ def _static_queue_pool(lanes: list[_ChartLane]) -> dict[str, object]: return pool +def _default_pool_pins_error(chart_lane: str, pins: Mapping[str, int]) -> UnrealizablePlanError: + identity, index = min(pins.items(), key=lambda item: item[1]) + return UnrealizablePlanError( + f"$.identities[{index}].warmth", + f"pins {identity!r} on chart lane {chart_lane!r}, but the gateway keeps no static queue pool for " + f"the {DEFAULT_QUEUE_POOL!r} queue pool, so its pinnedModels have no carrier; place pinned " + "identities on a named queuePool", + ) + + def _require_no_default_pool_pins(lanes: Iterable[_ChartLane]) -> None: for lane in lanes: if lane.pins: - identity, index = min(lane.pins.items(), key=lambda item: item[1]) - raise UnrealizablePlanError( - f"$.identities[{index}].warmth", - f"pins {identity!r} on chart lane {lane.name!r}, but the gateway keeps no static queue pool for " - f"the {DEFAULT_QUEUE_POOL!r} queue pool, so its pinnedModels have no carrier; place pinned " - "identities on a named queuePool", - ) + raise _default_pool_pins_error(lane.name, lane.pins) + + +def plan_refusals(plan: Plan) -> tuple[UnrealizablePlanError, ...]: + pins = _pins_by_lane(plan) + refusals: list[UnrealizablePlanError] = [] + for index, lane in enumerate(plan.lanes): + one_cell = _one_cell_error(f"$.lanes[{index}]", lane) + if one_cell is not None: + refusals.append(one_cell) + tokens = [value.strip(_GO_SPACE).lower() for value in (lane.queue_pool, lane.cell.machine_profile, lane.bundle)] + if tokens[0] == DEFAULT_QUEUE_POOL and pins.get(lane.name): + refusals.append(_default_pool_pins_error("/".join(tokens), pins[lane.name])) + refusals.extend(_warmth_errors(plan)) + return tuple(refusals) def helm_values(plan: Plan, parameters: ChartParameters | None = None) -> dict[str, object]: parameters = ChartParameters() if parameters is None else parameters lanes = _chart_lanes(plan, parameters) - _require_realizable_warmths(plan) + warmth = next(_warmth_errors(plan), None) + if warmth is not None: + raise warmth base = _enabled_base_bundles(parameters.base_pools) _require_lane_limit(lanes, base, parameters) by_profile: dict[str, list[_ChartLane]] = defaultdict(list) diff --git a/packages/sie_placement/src/sie_placement/keda.py b/packages/sie_placement/src/sie_placement/keda.py index 70a43e153..89006e133 100644 --- a/packages/sie_placement/src/sie_placement/keda.py +++ b/packages/sie_placement/src/sie_placement/keda.py @@ -14,11 +14,11 @@ from datetime import UTC, datetime from typing import Protocol -from sie_placement.schema import Plan, PlanLane, Warmth +from sie_placement.helm import plan_refusals +from sie_placement.schema import Plan, PlanLane from sie_placement.tuner import Observation, Settings, Step, step CARRIER_PREFIX = "placement-floor" -DEFAULT_QUEUE_POOL = "default" DEFAULT_MAX_POOL_FLOOR = 4 DEFAULT_MAX_POOLS = 64 MAX_POOL_NAME_LENGTH = 128 @@ -30,12 +30,6 @@ WARM_FLOOR = "warmFloor" RUNNERS = "runners" -UNREALIZED_WARMTHS = { - Warmth.BRIDGED: "the overlay configures no remote bridge to serve it while the lane boots", - Warmth.SNAPSHOT_COLD: "the chart restores no snapshot, so the lane would boot cold", - Warmth.SCHEDULED_WARM: "the chart keeps no warm window, so the lane would stay cold inside it", -} - _LANE_TOKEN = re.compile(r"[a-z0-9_-]{1,63}") _SELECTOR_FORBIDDEN = re.compile(r"[{}\r\n]") _LANE_LABELS = "pool, machine_profile, bundle" @@ -79,10 +73,6 @@ def canonical(cls, queue_pool: str, machine_profile: str, bundle: str) -> LaneKe def otel_lane(self) -> str: return f"{self.queue_pool}|{self.machine_profile}|{self.bundle}" - @property - def chart_lane(self) -> str: - return f"{self.queue_pool}/{self.machine_profile}/{self.bundle}" - @dataclass(frozen=True, slots=True) class Carrier: @@ -109,10 +99,6 @@ def carriers( ) result: list[Carrier] = [] first_index: dict[str, int] = {} - pins: dict[str, dict[str, int]] = {} - for index, entry in enumerate(plan.identities): - if entry.warmth is Warmth.PINNED: - pins.setdefault(entry.lane, {})[entry.identity] = index for index, lane in enumerate(plan.lanes): path = f"$.lanes[{index}]" key = LaneKey.canonical(lane.queue_pool, lane.cell.machine_profile, lane.bundle) @@ -121,14 +107,6 @@ def carriers( f"{path}: queuePool, cell.machineProfile and bundle must each be 1 to 63 letters, digits, '_' or '-'" ) continue - pinned = pins.get(lane.name) - if key.queue_pool == DEFAULT_QUEUE_POOL and pinned: - identity, pin = min(pinned.items(), key=lambda item: item[1]) - problems.append( - f"$.identities[{pin}].warmth: pins {identity!r} on chart lane {key.chart_lane!r}, but the gateway " - f"keeps no static queue pool for the {DEFAULT_QUEUE_POOL!r} queue pool, so its pinnedModels have no " - "carrier; place pinned identities on a named queuePool" - ) if lane.envelope.floor_max > max_pool_floor: problems.append( f"{path}.envelope.floorMax: {lane.envelope.floor_max} exceeds the gateway's " @@ -143,14 +121,7 @@ def carriers( if earlier != index: problems.append(f"{path}: needs the same floor-carrier pool {carrier.name!r} as $.lanes[{earlier}]") result.append(carrier) - for index, entry in enumerate(plan.identities): - reason = UNREALIZED_WARMTHS.get(entry.warmth) - if reason is not None: - problems.append( - f"$.identities[{index}].warmth: is {entry.warmth.value!r} for {entry.identity!r} on lane " - f"{entry.lane!r}, which the sie-cluster chart does not realize: {reason}; plan the identity pinned, " - "lazy-resident or cold-with-bound" - ) + problems.extend(str(refusal) for refusal in plan_refusals(plan)) if problems: raise PlanRefusedError(problems) return tuple(result) diff --git a/packages/sie_placement/tests/test_helm.py b/packages/sie_placement/tests/test_helm.py index e45c0b9d5..4e82910d5 100644 --- a/packages/sie_placement/tests/test_helm.py +++ b/packages/sie_placement/tests/test_helm.py @@ -18,6 +18,7 @@ chart_pools, helm_values, lane_token, + plan_refusals, ) from sie_placement.report import render from sie_placement.schema import ( @@ -870,3 +871,26 @@ def test_a_planner_estate_with_several_pinned_lanes_emits_a_named_queue_pool_for ) assert replanned.plan == result.plan assert helm_values(replanned.plan, ChartParameters(fallback_replicas=PACKING_PARAMETERS.fallback_floor)) == values + + +def test_plan_refusals_list_every_refusal_helm_values_makes_whatever_the_chart_parameters() -> None: + candidates = ( + plan_of(replace(lane("a", floor_min=0), fallback_cells=(A100,))), + plan_of(lane("a", queue_pool="Default")), + plan_of(lane("a", floor_min=0), warmths={"a": Warmth.SNAPSHOT_COLD}), + ) + for candidate in candidates: + (only,) = plan_refusals(candidate) + assert str(only) == str(refusal(candidate)) + combined = plan_of( + replace(lane("a", floor_min=0), fallback_cells=(A100,)), + lane("b", queue_pool="default", cell=A100), + lane("c", floor_min=0, bundle="c"), + warmths={"c": Warmth.BRIDGED}, + ) + assert [error.path for error in plan_refusals(combined)] == [ + "$.lanes[0].fallbackCells", + "$.identities[1].warmth", + "$.identities[2].warmth", + ] + assert plan_refusals(golden_plan()) == () diff --git a/packages/sie_placement/tests/test_keda.py b/packages/sie_placement/tests/test_keda.py index 3fa82a226..f479c3a4a 100644 --- a/packages/sie_placement/tests/test_keda.py +++ b/packages/sie_placement/tests/test_keda.py @@ -12,7 +12,7 @@ from urllib.parse import parse_qs, urlsplit import pytest -from sie_placement import keda +from sie_placement import helm, keda from sie_placement.schema import Plan from sie_placement.tuner import Action @@ -223,7 +223,7 @@ def test_a_pin_on_the_default_queue_pool_is_refused_as_the_helm_emitter_refuses_ assert pinned.name == "placement-floor-embed-l4-default" -@pytest.mark.parametrize("warmth", sorted(keda.UNREALIZED_WARMTHS)) +@pytest.mark.parametrize("warmth", sorted(helm.UNREALIZED_WARMTHS)) def test_a_warmth_the_chart_does_not_realize_is_refused_as_the_helm_emitter_refuses_it(warmth): document = plan_document() document["identities"][2]["warmth"] = warmth.value @@ -231,7 +231,7 @@ def test_a_warmth_the_chart_does_not_realize_is_refused_as_the_helm_emitter_refu keda.carriers(Plan.from_dict(document)) assert raised.value.problems == ( f"$.identities[2].warmth: is {warmth.value!r} for 'Qwen/Qwen3-8B' on lane 'default-a100-80gb-generation', " - f"which the sie-cluster chart does not realize: {keda.UNREALIZED_WARMTHS[warmth]}; plan the identity " + f"which the sie-cluster chart does not realize: {helm.UNREALIZED_WARMTHS[warmth]}; plan the identity " "pinned, lazy-resident or cold-with-bound", ) @@ -245,12 +245,37 @@ def test_every_problem_of_a_refused_plan_is_reported(): with pytest.raises(keda.PlanRefusedError) as raised: keda.carriers(Plan.from_dict(document)) assert [problem.split(":", 1)[0] for problem in raised.value.problems] == [ - "$.identities[0].warmth", "$.lanes[1].envelope.floorMax", + "$.identities[0].warmth", "$.identities[2].warmth", ] +def test_a_lane_with_fallback_cells_is_refused_as_the_helm_emitter_refuses_it(): + document = plan_document() + document["lanes"][1]["fallbackCells"] = [{"machineProfile": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3"}] + plan = Plan.from_dict(document) + with pytest.raises(keda.PlanRefusedError) as raised: + keda.carriers(plan) + with pytest.raises(helm.UnrealizablePlanError) as emitted: + helm.helm_values(plan, helm.ChartParameters(fallback_replicas=1)) + assert raised.value.problems == (str(emitted.value),) + assert raised.value.problems[0].startswith( + "$.lanes[1].fallbackCells: lists 'h100' with 'NVIDIA-H100-80GB-HBM3' after the lane's cell" + ) + + +def test_the_tuner_refuses_with_the_helm_emitter_s_own_words(): + document = plan_document() + document["identities"][0]["warmth"] = "pinned" + plan = Plan.from_dict(document) + with pytest.raises(keda.PlanRefusedError) as raised: + keda.carriers(plan) + with pytest.raises(helm.UnrealizablePlanError) as emitted: + helm.helm_values(plan, helm.ChartParameters(fallback_replicas=1)) + assert raised.value.problems == (str(emitted.value),) + + def test_the_series_queries_read_each_lane_from_fresh_gateway_producers_and_the_worker_series(): capacity_fresh = ( "and on(producer_instance, collector_generation) " From 4799e51872a238e7af25813876d41aa9328a84b5 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:15:14 +0100 Subject: [PATCH 22/40] fix(placement): charge the shared quota at envelope maxima - The shared quota charges every lane at its envelope maximum, the most a tuner may hold warm, together with the accelerators held elsewhere. Warm capacity is held continuously, so it must fit the quota that environments share. A burst above the floors is bounded by each environment's own total and per-type ceilings, environments sharing one accelerator quota rarely burst together, and a burst that meets the quota shows as a measured placement wait, which the planner adds to a cold come-alive. - The total and per-type ceilings still charge every lane at maxContainers, and a plan that does not fit is still refused whole. - The report gives peakAccelerators for total and type ceilings only, and a shared-quota refusal names the lanes whose envelope maxima it charges. --- packages/sie_placement/README.md | 29 +++++--- .../src/sie_placement/planner.py | 3 +- .../sie_placement/tests/test_availability.py | 72 +++++++++++-------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 4e39543e6..016b2eb56 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -117,14 +117,20 @@ obtain. Each field is optional, and at least one is declared: | `sharedQuota` | A quota of `accelerators` that this environment shares with others, which hold `heldElsewhere` of it | Every container holds the `gpuCount` accelerators of its lane's container -shape. Each ceiling must hold every lane it covers at its container ceiling, -`maxContainers` containers, at once, as a deployment compiler that enforces the -ceiling charges it, and so at its envelope maximum, `floorMax` containers, too. -`total` and `sharedQuota` cover every lane, and `sharedQuota` holds them -together with its `heldElsewhere` accelerators. A lane with fallback cells -counts against the ceiling of every product it names, and once against `total` -and `sharedQuota`. The planner never lowers a lane's `maxContainers` or -`floorMax` to fit a capacity document: a plan that does not fit is refused. +shape. `total` and each of `types` must hold every lane they cover at its +container ceiling, `maxContainers` containers, at once, as a deployment +compiler that enforces the ceiling charges it, and so at its envelope maximum, +`floorMax` containers, too. `sharedQuota` must hold every lane at its envelope +maximum, the most a tuner may hold warm, together with its `heldElsewhere` +accelerators. Warm capacity is held continuously, so it must fit the quota the +environments share. A burst above the floors is bounded by each environment's +own `total` and `types`, environments that share a quota rarely burst +together, and a burst that meets the shared quota shows as a measured +placement wait, which the planner adds to a cold come-alive. A lane with +fallback cells counts against the ceiling of every product it names, and once +against `total` and `sharedQuota`. The planner never lowers a lane's +`maxContainers` or `floorMax` to fit a capacity document: a plan that does not +fit is refused whole. ## Envelope @@ -345,9 +351,10 @@ The report renders as a `placement-report` document: - `capacity`: for each ceiling of the `capacity` document, its `scope` (`total`, `type` with its `gpuProduct`, or `shared-quota` with `heldElsewhere`), the `ceiling`, the `accelerators` the plan's envelope - maxima hold against it, the `peakAccelerators` its lanes hold at - `maxContainers`, both with `heldElsewhere` for `shared-quota`, the `lanes` - that hold them, and whether it is `exceeded`. + maxima hold against it, with `heldElsewhere` for `shared-quota`, for `total` + and `type` the `peakAccelerators` its lanes hold at `maxContainers`, the + `lanes` that hold them, and whether it is `exceeded`. `total` and `type` are + charged at `peakAccelerators`, and `shared-quota` at `accelerators`. - `lanes`: each lane's hourly cost, its monthly cost, its floor's monthly cost and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 505dff824..48e5ff752 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1176,9 +1176,8 @@ def held(lanes: Iterable[PlanLane]) -> tuple[int, int, tuple[str, ...]]: scope=CapacityScope.SHARED_QUOTA, ceiling=quota.accelerators, accelerators=total + quota.held_elsewhere, - peak_accelerators=peak + quota.held_elsewhere, held_elsewhere=quota.held_elsewhere, - lanes=names, + lanes=tuple(lane.name for lane in plan.lanes if lane.envelope.floor_max), ) ) return tuple(uses) diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index 0304772b6..8874bb72c 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -534,7 +534,7 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a (CapacityScope.TOTAL, None, 4, 4, 4, ("org-a-l4", "org-b-a100-80gb")), (CapacityScope.TYPE, A100.gpu_product, 2, 2, 2, ("org-b-a100-80gb",)), (CapacityScope.TYPE, "NVIDIA-L4", 2, 2, 2, ("org-a-l4",)), - (CapacityScope.SHARED_QUOTA, None, 6, 6, 6, ("org-a-l4", "org-b-a100-80gb")), + (CapacityScope.SHARED_QUOTA, None, 6, 6, None, ("org-a-l4", "org-b-a100-80gb")), ] assert not any(use.exceeded for use in report.capacity) assert report.to_dict()["capacity"][0] == { @@ -549,7 +549,6 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a "scope": "shared-quota", "ceiling": 6, "accelerators": 6, - "peakAccelerators": 6, "heldElsewhere": 2, "lanes": ["org-a-l4", "org-b-a100-80gb"], "exceeded": False, @@ -573,14 +572,9 @@ def test_capacity_reports_what_the_envelope_maxima_and_container_ceilings_hold_a ), ( Capacity(shared_quota=SharedQuota(accelerators=6, held_elsewhere=3)), - "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " "which with 3 held elsewhere exceed the shared quota of 6", ), - ( - Capacity(shared_quota=SharedQuota(accelerators=10, held_elsewhere=3)), - "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, " - "which with 3 held elsewhere exceed the shared quota of 10", - ), ], ) def test_a_plan_whose_lanes_exceed_a_ceiling_is_refused_and_says_why(capacity, message): @@ -614,25 +608,42 @@ def test_the_container_ceilings_bind_where_the_envelope_maxima_fit(): assert [(use.scope, use.accelerators, use.peak_accelerators) for use in caught.value.exceeded] == [ (CapacityScope.TOTAL, 4, 8), (CapacityScope.TYPE, 2, 4), - (CapacityScope.SHARED_QUOTA, 7, 11), ] result = two_lanes(capacity=capacity, max_containers=2) assert [planned.max_containers for planned in result.plan.lanes] == [2, 2] -def test_a_refused_plan_keeps_every_container_ceiling_it_would_need(): - capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=5)) +def test_the_shared_quota_holds_the_envelope_maxima_where_the_container_ceilings_exceed_it(): + capacity = Capacity(total=8, shared_quota=SharedQuota(accelerators=7, held_elsewhere=3)) + result = two_lanes(capacity=capacity) + assert [(planned.max_containers, planned.envelope.floor_max) for planned in result.plan.lanes] == [(4, 2), (4, 2)] + assert [(use.scope, use.accelerators, use.peak_accelerators, use.exceeded) for use in result.report.capacity] == [ + (CapacityScope.TOTAL, 4, 8, False), + (CapacityScope.SHARED_QUOTA, 7, None, False), + ] + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=replace(capacity, total=7)) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.TOTAL] + with pytest.raises(CapacityExceededError) as caught: + two_lanes(capacity=replace(capacity, shared_quota=SharedQuota(accelerators=6, held_elsewhere=3))) + assert [use.scope for use in caught.value.exceeded] == [CapacityScope.SHARED_QUOTA] + + +def test_a_refused_plan_keeps_every_container_ceiling_and_envelope_maximum_it_would_need(): + capacity = Capacity(total=5, shared_quota=SharedQuota(accelerators=3)) with pytest.raises(CapacityExceededError) as caught: two_lanes(capacity=capacity) refused = caught.value.result.plan - assert [(planned.name, planned.max_containers) for planned in refused.lanes] == [ - ("org-a-l4", 4), - ("org-b-a100-80gb", 4), + assert [(planned.name, planned.max_containers, planned.envelope.floor_max) for planned in refused.lanes] == [ + ("org-a-l4", 4, 2), + ("org-b-a100-80gb", 4, 2), ] assert refused == two_lanes().plan - for use in caught.value.exceeded: - assert use.lanes == ("org-a-l4", "org-b-a100-80gb") - assert "org-a-l4, org-b-a100-80gb hold 8 accelerators" in use.describe() + assert [use.describe() for use in caught.value.exceeded] == [ + "the container ceilings of org-a-l4, org-b-a100-80gb hold 8 accelerators, above the total ceiling of 5", + "the envelope maxima of org-a-l4, org-b-a100-80gb hold 4 accelerators, " + "which with 0 held elsewhere exceed the shared quota of 3", + ] def test_a_candidate_list_lane_counts_against_every_member_type(): @@ -658,10 +669,12 @@ def test_each_container_holds_its_gpu_count(): def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_ceiling(): - result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=4)) + capacity = Capacity(total=4, shared_quota=SharedQuota(accelerators=0)) + result = run(one(600), [receipt()], [attempt(boot=60.0)], capacity=capacity) assert result.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ - (0, 4, ("org-model-l4",)) + (0, 4, ("org-model-l4",)), + (0, None, ()), ] with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): run(one(600), [receipt()], [attempt(boot=60.0)], capacity=Capacity(total=3)) @@ -669,18 +682,14 @@ def test_a_zero_floor_lane_holds_no_envelope_maximum_but_holds_its_container_cei def test_a_loosening_whose_plan_the_ceilings_would_refuse_saves_nothing(): receipts, boots = [receipt()], [attempt(boot=240.0)] - current = plan_of( - [replace(lane("org-model-l4", buffer=1), max_containers=2)], {IDENTITY: ("org-model-l4", Warmth.PINNED)} - ) - free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, current=current) - assert free.plan == current + free = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3) assert [lever.permitted_bound_s for lever in free.report.come_alive_levers] == [240] capacity = Capacity(shared_quota=SharedQuota(accelerators=2)) - bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=current) + bounded = run(one(60), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity) assert bounded.plan == free.plan assert bounded.report.come_alive_levers == () assert bounded.report.identity(IDENTITY).next_cheaper.saving == 0 - with pytest.raises(CapacityExceededError, match="the container ceilings of org-model-l4 hold 4 accelerators"): + with pytest.raises(CapacityExceededError, match="the envelope maxima of org-model-l4 hold 3 accelerators"): run(one(240), receipts, boots, buffer_containers=1, fallback_floor=3, capacity=capacity, current=free.plan) @@ -853,9 +862,14 @@ def failure_rate(product): peak = sum(planned.max_containers * planned.container_shape.gpu_count for planned in lanes) elsewhere = use.held_elsewhere or 0 assert use.accelerators == held + elsewhere - assert use.peak_accelerators == peak + elsewhere - assert use.exceeded is (peak + elsewhere > use.ceiling) - assert use.lanes == tuple(planned.name for planned in lanes) + if use.scope is CapacityScope.SHARED_QUOTA: + assert use.peak_accelerators is None + assert use.exceeded is (held + elsewhere > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes if planned.envelope.floor_max) + else: + assert use.peak_accelerators == peak + assert use.exceeded is (peak > use.ceiling) + assert use.lanes == tuple(planned.name for planned in lanes) assert (error is not None) is any(use.exceeded for use in result.report.capacity) if error is not None: assert error.exceeded == tuple(use for use in result.report.capacity if use.exceeded) From 2601bb61b8d7af4a3203bd6515e918dfa160c9b1 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:19:58 +0100 Subject: [PATCH 23/40] test(placement): hold a shared lane to the shared quota at its envelope maximum - A shared lane holds its container ceiling once against the total and its GPU type, and its envelope maximum once against the shared quota, so a packing fits a quota the separate lanes' envelope maxima would exceed. --- packages/sie_placement/tests/test_packing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index ecc2f36d5..2129301cd 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -655,11 +655,11 @@ def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(uns assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) -def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): +def test_a_shared_lane_holds_its_container_ceiling_and_envelope_maximum_once_against_every_ceiling(): capacity = Capacity( total=4, types=(TypeCeiling(gpu_product=L4.gpu_product, accelerators=4),), - shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + shared_quota=SharedQuota(accelerators=3, held_elsewhere=2), ) service_levels = levels(**{HOST: 0, GUEST: 60}) result = run(service_levels, host_and_guest(), [load(GUEST)], capacity=capacity) @@ -667,15 +667,15 @@ def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ (1, 4, ("org-host-l4",)), (1, 4, ("org-host-l4",)), - (3, 6, ("org-host-l4",)), + (3, None, ("org-host-l4",)), ] with pytest.raises(CapacityExceededError) as caught: run(service_levels, host_and_guest(guest_footprint=8000), [load(GUEST)], capacity=capacity) assert [use.describe() for use in caught.value.exceeded] == [ "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, above the total ceiling of 4", "the container ceilings of org-guest-l4, org-host-l4 hold 8 NVIDIA-L4 accelerators, above the type ceiling of 4", - "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, " - "which with 2 held elsewhere exceed the shared quota of 6", + "the envelope maxima of org-guest-l4, org-host-l4 hold 2 accelerators, " + "which with 2 held elsewhere exceed the shared quota of 3", ] From 78fe4e31696174816a97ad052f9d2ea2162fdf46 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:30:05 +0100 Subject: [PATCH 24/40] fix(placement): check capacity counts as integers in documents built in code - total, each type ceiling's accelerators and the shared quota's accelerators and heldElsewhere go through the integer check, so a capacity document built in code refuses booleans and floats as parsing does. --- packages/sie_placement/src/sie_placement/schema.py | 8 ++++---- packages/sie_placement/tests/test_schema.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index 7f0f83acc..b445aba45 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -949,7 +949,7 @@ class TypeCeiling: def __post_init__(self) -> None: _require_identifier("gpuProduct", self.gpu_product) - _require_at_least("accelerators", self.accelerators, 0) + _require_count("accelerators", self.accelerators, 0) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -966,8 +966,8 @@ class SharedQuota: held_elsewhere: int = 0 def __post_init__(self) -> None: - _require_at_least("accelerators", self.accelerators, 0) - _require_at_least("heldElsewhere", self.held_elsewhere, 0) + _require_count("accelerators", self.accelerators, 0) + _require_count("heldElsewhere", self.held_elsewhere, 0) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -989,7 +989,7 @@ class Capacity: shared_quota: SharedQuota | None = None def __post_init__(self) -> None: - _require_at_least("total", self.total, 0) + _require_count("total", self.total, 0) _require_unique("types", "gpuProduct", (ceiling.gpu_product for ceiling in self.types)) if self.total is None and not self.types and self.shared_quota is None: raise SchemaError("", "must declare total, types or sharedQuota") diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index b828daece..454ad6c4b 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -187,6 +187,9 @@ def test_numbers_built_in_code_must_be_numbers(value, described): (lambda value: ServiceLevel(come_alive_p95_s=value), "comeAliveP95S"), (lambda value: Envelope(floor_min=0, floor_max=value), "floorMax"), (lambda value: DemandHour(start=datetime(2026, 1, 1, tzinfo=UTC), requests=value), "requests"), + (lambda value: Capacity(total=value), "total"), + (lambda value: TypeCeiling(gpu_product="NVIDIA-L4", accelerators=value), "accelerators"), + (lambda value: SharedQuota(accelerators=8, held_elsewhere=value), "heldElsewhere"), ], ) @pytest.mark.parametrize(("value", "described"), [(1.5, "number 1.5"), (2.0, "number 2.0"), (True, "boolean true")]) From b8ff7d86e88b00f337e6b0506f164308e699e826 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:31:07 +0100 Subject: [PATCH 25/40] fix(placement): check memory sizes as integers in documents built in code - A receipt's footprintMiB and workingMemoryMiB, a boot row's footprintMiB and a machine profile's gpuMemoryMiB go through the integer check, so a document built in code refuses booleans and floats as parsing does. --- .../sie_placement/src/sie_placement/schema.py | 8 +++--- packages/sie_placement/tests/test_schema.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index ba9b8404a..4909e13c2 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -644,8 +644,8 @@ class Receipt: def __post_init__(self) -> None: _require_identifier("identity", self.identity) _require_count("concurrency", self.concurrency, 1) - _require_at_least("footprintMiB", self.footprint_mib, 1) - _require_at_least("workingMemoryMiB", self.working_memory_mib, 0) + _require_count("footprintMiB", self.footprint_mib, 1) + _require_count("workingMemoryMiB", self.working_memory_mib, 0) for index, resident in enumerate(self.co_residents): _require_identifier(f"coResidents[{index}]", resident) if resident == self.identity: @@ -742,7 +742,7 @@ def __post_init__(self) -> None: raise SchemaError("bootSeconds", "is required for an attempt that was placed") _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) - _require_at_least("footprintMiB", self.footprint_mib, 1) + _require_count("footprintMiB", self.footprint_mib, 1) if self.outcome is not BootOutcome.SUCCEEDED: _require_absent("footprintMiB", self.footprint_mib, "for an attempt that did not succeed") @@ -798,7 +798,7 @@ def __post_init__(self) -> None: _require_at_least("acceleratorPerHour", self.accelerator_per_hour, 0) _require_at_least("corePerHour", self.core_per_hour, 0) _require_at_least("memoryGiBPerHour", self.memory_gib_per_hour, 0) - _require_at_least("gpuMemoryMiB", self.gpu_memory_mib, 1) + _require_count("gpuMemoryMiB", self.gpu_memory_mib, 1) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index 3aa73bf9d..6248bab36 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -4,6 +4,7 @@ import json import math from collections.abc import Callable +from dataclasses import replace from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -190,6 +191,17 @@ def test_numbers_built_in_code_must_be_numbers(value, described): (lambda value: Capacity(total=value), "total"), (lambda value: TypeCeiling(gpu_product="NVIDIA-L4", accelerators=value), "accelerators"), (lambda value: SharedQuota(accelerators=8, held_elsewhere=value), "heldElsewhere"), + ( + lambda value: MachineProfilePrice( + name="l4", + gpu_product="NVIDIA-L4", + accelerator_per_hour=0, + core_per_hour=0, + memory_gib_per_hour=0, + gpu_memory_mib=value, + ), + "gpuMemoryMiB", + ), ], ) @pytest.mark.parametrize(("value", "described"), [(1.5, "number 1.5"), (2.0, "number 2.0"), (True, "boolean true")]) @@ -566,6 +578,21 @@ def test_boot_evidence_records_the_footprint_of_a_load_that_succeeded(): ) +@pytest.mark.parametrize( + ("parse", "name", "field"), + [ + (Receipt.from_dict, "receipt-encode.json", "footprint_mib"), + (Receipt.from_dict, "receipt-encode.json", "working_memory_mib"), + (BootEvidence.from_dict, "boot-evidence.json", "footprint_mib"), + ], +) +def test_memory_built_in_code_must_be_integers(parse, name, field): + document = parse(load(name)) + with pytest.raises(SchemaError) as caught: + replace(document, **{field: 1.5}) + assert caught.value.message == "must be an integer, got number 1.5" + + def test_machine_profiles_may_state_the_memory_of_their_card(): data = load("price-list.json") data["machineProfiles"][0]["gpuMemoryMiB"] = 23034 From cb0ed0a704c5ca26e82740b419d3a2f350665b28 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:32:50 +0100 Subject: [PATCH 26/40] docs(placement): say a derived queue pool is the slug of the lane name - A lane whose current plan puts it on the default pool and is named Shared_Lane derives the pool shared-lane, which a test now pins. --- packages/sie_placement/README.md | 13 ++++++++----- packages/sie_placement/tests/test_planner.py | 7 +++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 0aff76c9f..f995bd7ae 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -103,11 +103,14 @@ The planner gives each lane that pins at least one identity a named queue pool of its own, since the chart carries a lane's pinned set on a static queue pool and the default queue pool carries no pins. A lane the current plan already puts on a named queue pool keeps it, so planning again leaves the pool as it -is. Otherwise the pool is derived from the lane's `name`: the name itself, cut -to 63 characters with a digest of the name appended when it is longer or when -another lane of the plan, `default`, or the parameters' `queue_pool` already -has it. A lane on which every identity is lazy-resident or cold is on the -parameters' `queue_pool`. Identities that share a lane share its queue pool. +is. Otherwise the pool is derived from the lane's `name`: its slug, which +lowercases the name, replaces each run of characters outside `a-z` and `0-9` +with a hyphen and drops leading and trailing hyphens, or `lane` when nothing +remains. The slug is cut to 63 characters with a digest of the name appended +when it is longer or when another lane of the plan, `default`, or the +parameters' `queue_pool` already has it. A lane on which every identity is +lazy-resident or cold is on the parameters' `queue_pool`. Identities that share +a lane share its queue pool. `fallbackCells` is an optional ordered candidate list: cells, after `cell`, that the lane's containers may be placed on when `cell` cannot be placed. No diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index f5a73fcb3..9d06386bc 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1566,6 +1566,13 @@ def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_ } +def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): + current = single_lane_plan(name="Shared_Lane") + current = replace(current, lanes=(replace(current.lanes[0], queue_pool="default"),)) + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), [receipt()], prices=L4_ONLY, current=current) + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == {"Shared_Lane": "shared-lane"} + + def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): digest = hashlib.sha256(IDENTITY.encode()).hexdigest() first = single_lane_plan(name="org-model-l4", identity="org/kept") From 21b21d26ba037f545a44c1795a195efc964dbd30 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:39:50 +0100 Subject: [PATCH 27/40] docs(placement): list the fallback-floor and queue-pool emit conditions - A planner plan emits only when no zero-floor lane's nonzero floorMax is below the chart's fallback floor, which holds when fallback_replicas is at most the planner's fallback_floor, and when the lanes on a non-default queue_pool share one floorMin and, with a nonzero floorMin, one bundle. --- packages/sie_placement/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index a4a107382..53cbe863d 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -619,13 +619,17 @@ These rules bound which plans emit. A worker group carries one queue pool, a named queue pool carries one floor and one pinned set, and the `default` queue pool carries no pins, so a pinned lane needs a named queue pool and a machine profile of its own, and two pinned lanes on one GPU product need two machine -profiles. Such a lane may pin several identities and host lazy residents -beside them. The planner gives every lane that pins an identity a named queue -pool of its own, places every other lane on the queue pool its parameters name, +profiles. Such a lane may pin several identities and host lazy residents beside +them. The planner gives every lane that pins an identity a named queue pool of +its own, places every other lane on the queue pool its parameters name, `default` unless set, and may place several identities on one lane. Its plan therefore emits only when no two lanes share a machine profile and bundle, no -lane that pins an identity shares its machine profile with another lane, no lane -has fallback cells and no identity is `snapshot-cold`. +lane that pins an identity shares its machine profile with another lane, no +lane has fallback cells, no identity is `snapshot-cold`, no zero-floor lane's +nonzero `floorMax` is below the chart's fallback floor, which holds when the +chart's `fallback_replicas` is at most the planner's `fallback_floor`, and, +when the planner's `queue_pool` is not `default`, the lanes on it share one +`floorMin` and, with a nonzero `floorMin`, one bundle. ## Tests From 1a23347fff7e8c2ef1612c9643a9f70df0c37f59 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:42:50 +0100 Subject: [PATCH 28/40] docs(helm): state where applier failures show and how to rotate its admin token - A lapsed carrier returns the lane to the largest floor its chart and its other pools set, not always to the chart floor. - A failed Prometheus query goes to standard error and shows as null series in the audit rows; the audit row's error records a failed pool call. - The gateway reads its single admin token at start, so an admin-token rotation restarts the gateway and then the applier. --- deploy/helm/sie-cluster/README.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 013531dec..c4e7abc50 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -549,7 +549,8 @@ ScaledObject triggers are the same with or without the applier. - **Apply mode** (`placementApplier.apply: true`) upserts each lane's carrier on every tick, which renews the carrier's lease and sets its floor. When the applier stops, each carrier lapses 1200 seconds after its last write by - default, the gateway removes it, and the lane returns to its chart floor. + default, the gateway removes it, and the lane returns to the largest floor + its chart and its other pools set. The render fails unless: @@ -582,8 +583,9 @@ The applier itself refuses a plan whose `floorMax` exceeds the gateway's per-pool floor limit, 4 by default, or whose lanes need more carriers than the gateway admits, 64 by default. A refusal, an unreadable token file or an invalid flag ends the container, which then shows as `CrashLoopBackOff`. A -failed Prometheus query or pool call fails only its tick, which the audit row -records. +failed Prometheus query or pool call fails only its tick. The applier prints a +failed query to standard error, and the audit rows show each series it could +not read as `null`. The audit row's `error` records a failed pool call. #### Applier permissions @@ -606,16 +608,21 @@ Secret can still read it, so the Secret read limits under [Who holds which token](#who-holds-which-token) apply. The applier reads both token files once, when it starts. After you rotate the -gateway client token or the gateway admin token, restart it: +gateway client token, restart it as step 2 of +[rotating the gateway token](#gateway-token) says. The gateway also reads its +admin token once, when it starts, and accepts one at a time, so after you +rotate the gateway admin token, restart the gateway and then the applier: ```bash +kubectl rollout restart deployment/-gateway -n kubectl rollout restart deployment/-placement-applier -n ``` -Until it restarts, the gateway refuses its pool reads or writes and the pod -keeps running: each tick's audit rows record the failures, and in apply mode -each carrier lapses 1200 seconds after its last write and the lane returns to -its chart floor. +The cutover is complete once both run with the new token. Until then the +gateway refuses the applier's pool reads or writes and the pod keeps running: +each tick's audit rows record the failures, and in apply mode each carrier +lapses 1200 seconds after its last write and the lane returns to the largest +floor its chart and its other pools set. The Deployment runs one replica with the `Recreate` strategy, so at most one applier writes carriers, also during a rollout. Its pod runs as user and group From 1e781a014bc36b51467090571faf1012e7125e88 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:43:29 +0100 Subject: [PATCH 29/40] fix(placement): group plan lanes into chart lanes without regard to case - sharedChartLanes compares queuePool, machineProfile and bundle in lowercase, as the chart canonicalizes lane tokens, so two plan lanes whose tokens differ only in case are reported as one chart lane. --- packages/sie_placement/README.md | 9 +++++---- .../sie_placement/src/sie_placement/planner.py | 2 +- packages/sie_placement/tests/test_planner.py | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 7a8be6e86..768503b2a 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -89,9 +89,10 @@ Helm chart: `queuePool`, the `machineProfile` of its `cell`, and `bundle`. override. It is neither the `workers.pools` map key, which names a worker group, nor an SDK logical pool. `bundle` is a key of that worker group's `bundles` map. `name` is the lane's handle inside the plan, and each identity -refers to its lane by that name. Two plan lanes may carry the same three -tokens, and so name one chart lane, which cannot give them separate envelopes -or concurrency; the planner's report lists every such chart lane. +refers to its lane by that name. The chart compares the three tokens without +regard to case. Two plan lanes may carry the same three tokens, and so name one +chart lane, which cannot give them separate envelopes or concurrency; the +planner's report lists every such chart lane. ## Envelope @@ -274,7 +275,7 @@ The report renders as a `placement-report` document: and `unpricedIdentities` lists residents whose demand has no receipt to price it on the lane. - `sharedChartLanes`: every chart lane, by `queuePool`, `machineProfile` and - `bundle`, that more than one plan lane names. + `bundle` in lowercase, that more than one plan lane names. - `measurementRequests`: options that lack evidence, ranked by the saving they could unlock. A profile without a receipt is bounded by price alone: one accelerator at its rate, kept warm or, where the come-alive bound allows, diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index b3e82d23b..8dfa39d35 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1109,7 +1109,7 @@ def _permitted(measured: float) -> int: def _shared_chart_lanes(plan: Plan) -> tuple[SharedChartLane, ...]: grouped: defaultdict[tuple[str, str, str], list[str]] = defaultdict(list) for lane in plan.lanes: - grouped[(lane.queue_pool, lane.cell.machine_profile, lane.bundle)].append(lane.name) + grouped[(lane.queue_pool.lower(), lane.cell.machine_profile.lower(), lane.bundle.lower())].append(lane.name) return tuple( SharedChartLane(queue_pool=pool, machine_profile=profile, bundle=bundle, lanes=tuple(sorted(names))) for (pool, profile, bundle), names in sorted(grouped.items()) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 0141707b1..00104fa88 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1394,6 +1394,20 @@ def test_plan_lanes_that_name_one_chart_lane_are_reported(): assert apart.report.shared_chart_lanes == () +def test_plan_lanes_whose_tokens_differ_only_in_case_name_one_chart_lane(): + service_levels = levels(("org/a", WARM), ("org/b", WARM)) + receipts = [receipt(identity=identity) for identity in ("org/a", "org/b")] + result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "Default"}) + assert {lane.name: lane.bundle for lane in result.plan.lanes} == {"org-a-l4": "Default", "org-b-l4": "default"} + (shared,) = result.report.shared_chart_lanes + assert shared.to_dict() == { + "queuePool": "default", + "machineProfile": "l4", + "bundle": "default", + "lanes": ["org-a-l4", "org-b-l4"], + } + + def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): digest = hashlib.sha256(IDENTITY.encode()).hexdigest() first = single_lane_plan(name="org-model-l4", identity="org/kept") From 8676b86ddfa7ef7f679d63c30a97a98bdfbb4a53 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:44:36 +0100 Subject: [PATCH 30/40] fix(placement): compare kept queue pools without regard to case - A pinning lane keeps its current named queue pool only when no other lane keeps it and it is neither default nor the parameters' queue_pool, each compared in lowercase as the chart canonicalizes lane tokens. A current pool spelled Default, or a second lane on Batch and batch, gets a derived pool instead. --- packages/sie_placement/README.md | 4 +++- .../src/sie_placement/planner.py | 6 +++--- packages/sie_placement/tests/test_planner.py | 21 ++++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 744c44b95..7de99ff95 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -104,7 +104,9 @@ The planner gives each lane that pins at least one identity a named queue pool of its own, since the chart carries a lane's pinned set on a static queue pool and the default queue pool carries no pins. A lane the current plan already puts on a named queue pool keeps it, so planning again leaves the pool as it -is. Otherwise the pool is derived from the lane's `name`: its slug, which +is, unless the pool is `default`, the parameters' `queue_pool` or a pool +another lane keeps, compared without regard to case as the chart compares them. +Otherwise the pool is derived from the lane's `name`: its slug, which lowercases the name, replaces each run of characters outside `a-z` and `0-9` with a hyphen and drops leading and trailing hyphens, or `lane` when nothing remains. The slug is cut to 63 characters with a digest of the name appended diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 68bff74e0..33df139e6 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1475,12 +1475,12 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: def queue_pools(self, lanes: Iterable[str], identities: Iterable[PlanIdentity]) -> dict[str, str]: pools = dict.fromkeys(lanes, self.parameters.queue_pool) pinning = sorted({entry.lane for entry in identities if entry.warmth is Warmth.PINNED}) - taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool} + taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool.lower()} for name in pinning: current = self.current_lanes.get(name) - if current is not None and current.queue_pool not in taken: + if current is not None and current.queue_pool.lower() not in taken: pools[name] = current.queue_pool - taken.add(current.queue_pool) + taken.add(current.queue_pool.lower()) for name in pinning: if pools[name] == self.parameters.queue_pool: pools[name] = _named_queue_pool(name, taken) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 057d1748b..1cc876fab 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1566,6 +1566,24 @@ def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_ } +def test_a_kept_queue_pool_is_compared_without_regard_to_case(): + kept = [single_lane_plan(name=name, identity=f"org/{name}") for name in ("first", "second", "third")] + pools = ("Batch", "batch", "Default") + current = Plan( + placement="test", + lanes=tuple(replace(entry.lanes[0], queue_pool=pool) for entry, pool in zip(kept, pools, strict=True)), + identities=tuple(identity for entry in kept for identity in entry.identities), + ) + pinned = ServiceLevel(come_alive_p95_s=0) + service_levels = levels(*((f"org/{name}", pinned) for name in ("first", "second", "third"))) + result = run(service_levels, [], prices=L4_ONLY, current=current) + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == { + "first": "Batch", + "second": "second", + "third": "third", + } + + def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): current = single_lane_plan(name="Shared_Lane") current = replace(current, lanes=(replace(current.lanes[0], queue_pool="default"),)) @@ -1576,7 +1594,8 @@ def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): def test_plan_lanes_whose_tokens_differ_only_in_case_name_one_chart_lane(): service_levels = levels(("org/a", WARM), ("org/b", WARM)) receipts = [receipt(identity=identity) for identity in ("org/a", "org/b")] - result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "Default"}) + boots = [boot(identity=identity) for identity in ("org/a", "org/b")] + result = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/a": "Default"}) assert {lane.name: lane.bundle for lane in result.plan.lanes} == {"org-a-l4": "Default", "org-b-l4": "default"} (shared,) = result.report.shared_chart_lanes assert shared.to_dict() == { From 5f3d1a653020d070cae81ae5699b8d4e0e46b46f Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:54:56 +0100 Subject: [PATCH 31/40] docs(placement): say when the planner requests footprints and what a profile without card memory blocks --- packages/sie_placement/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 7de99ff95..24606bb52 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -189,10 +189,15 @@ it had before. Where the identities that join a lane do not fit it together, they are admitted first fit in decreasing order of footprint, ties broken by identity, and the others are decided again without that lane. -Where a resident's footprint or working memory is not on file, the planner -asks that resident for it, under its own identity and warmth at the lane, in a -`footprint` measurement request. A lane on a profile without `gpuMemoryMiB` -hosts no identity beside another, and the report names that profile. +Where an identity would join a lane and a resident's footprint or working +memory is not on file, the planner asks that resident for it, under its own +identity and warmth at the lane, in a `footprint` measurement request. For a +lane the plan keeps, it asks only when a lazy resident is placed there. A kept +lane whose pinned residents have no footprint on file gets no `footprint` +request, so the report does not name every unmeasured shared lane. A profile +without `gpuMemoryMiB` takes no new identity beside another, and the report +names that profile. A lane the plan already shares on such a profile stays +shared. ## Envelope From c35ea60ac3fd41de0172ea73f5a42bdbea0f593a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:14:55 +0100 Subject: [PATCH 32/40] fix(placement): size a shared lane from its residents' demand together - A lane with a floor and several priced residents now costs the containers each hour needs for their requests together, each resident's requests taking the share of a container its priced receipt's rate gives them. Pricing each resident's demand against the floor on its own counted a container that both residents need as free, so a packing could look cheaper than the lane it makes. - A packing after which the lane's residents together need more containers than its maxContainers in some hour is not made, so two residents that each fill a one-container ceiling are not packed onto it. --- packages/sie_placement/README.md | 23 ++++++----- .../src/sie_placement/planner.py | 38 +++++++++++++++++-- packages/sie_placement/tests/test_packing.py | 27 +++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 24606bb52..f3e3701a1 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -375,7 +375,9 @@ plan total does not fall. otherwise pinned, which needs every pinned identity's co-residency receipt beside the others to meet its warm bounds. A move that would lower the total but lacks a footprint, a working memory, a load or a co-residency receipt is - not made: it is a measurement request whose `savingBound` is that fall. + not made: it is a measurement request whose `savingBound` is that fall. A + move after which the lane's residents together need more containers than + its `maxContainers` in some hour is not made either. 5. Steps 1 and 3 run again for the lanes packing changed. ### Pricing @@ -393,14 +395,17 @@ demand window to `hours_per_month`. every request: the active floor for `cooldown_s`, at most the whole hour. The active floor is `fallback_floor` capped by the ceiling, and at least one. -A lane costs its floor all month once, whoever it hosts, plus what each -resident's demand needs beyond the floor: on a lane with a floor, the -containers an hour needs above it, and on a zero-floor lane, the work and the -tails a cold warmth pays. A placed identity's demand is priced at its chosen -receipt, and a kept identity's at the slowest receipt it has for its lane's -cells, container shape and concurrency, under that lane's own ceiling. A lane -with fallback cells costs the hourly rate of its dearest cell, and a candidate -list's demand is priced at the slower of its two receipts. +A lane costs its floor all month once, whoever it hosts, plus what its +residents' demand needs beyond the floor. On a lane with a floor, that is the +containers an hour needs above it for the residents together: each resident's +requests in the hour take the share of a container that its priced receipt's +rate gives them, and the shares are summed, since residents scale together and +share the card under load. On a zero-floor lane, it is the work and the tails a +cold warmth pays for each resident. A placed identity's demand is priced at its +chosen receipt, and a kept identity's at the slowest receipt it has for its +lane's cells, container shape and concurrency, under that lane's own ceiling. A +lane with fallback cells costs the hourly rate of its dearest cell, and a +candidate list's demand is priced at the slower of its two receipts. ### Envelope diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 33df139e6..c6526e53a 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -522,13 +522,14 @@ def _boot_index(records: Iterable[BootEvidence], placements: Mapping[str, _Place return index -def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: +def _hourly_requests(demand: Demand) -> dict[str, dict[datetime, int]]: hours: defaultdict[str, Counter[datetime]] = defaultdict(Counter) for series in demand.series: for hour in series.hours: hours[series.identity][hour.start] += hour.requests return { - identity: tuple(count for _, count in sorted(counts.items()) if count) for identity, counts in hours.items() + identity: {start: count for start, count in sorted(counts.items()) if count} + for identity, counts in hours.items() } @@ -579,7 +580,8 @@ def __init__( active_floor=parameters.active_floor, cooldown_s=parameters.cooldown_s, ) - self.requests = _hourly_requests(demand) + self.requests_by_hour = _hourly_requests(demand) + self.requests = {identity: tuple(counts.values()) for identity, counts in self.requests_by_hour.items()} self.current_plan = current_plan self.current: dict[str, tuple[PlanIdentity, PlanLane]] = {} self.current_lanes: dict[str, PlanLane] = {} @@ -1643,6 +1645,20 @@ def stay_beside(self, choices: dict[str, _Choice], originals: Mapping[str, _Choi choices, layout, total = trial, trial_layout, trial_total return choices + def lane_needs(self, rates: Mapping[str, float]) -> list[int]: + load: Counter[datetime] = Counter() + for identity, rate in rates.items(): + for start, count in self.requests_by_hour.get(identity, {}).items(): + load[start] += count / (rate * SECONDS_PER_HOUR) + return [math.ceil(round(containers, 9)) for containers in load.values()] + + def lane_rates(self, layout: _Layout, name: str) -> dict[str, float]: + return { + entry.identity: receipt.throughput.requests_per_second + for entry in layout.plan.identities + if entry.lane == name and (receipt := layout.receipts.get(entry.identity)) is not None + } + def riding_cost(self, lane: PlanLane, identity: str, receipt: Receipt) -> float: window = replace(self.window, ceiling=lane.max_containers) requests = self.requests.get(identity, ()) @@ -1830,6 +1846,10 @@ def floor_cost(identity: str) -> float: }, } trial_layout = self.assemble(trial) + name = trial_layout.names[identity] + ceiling = next(lane.max_containers for lane in trial_layout.plan.lanes if lane.name == name) + if any(need > ceiling for need in self.lane_needs(self.lane_rates(trial_layout, name))): + continue trial_total = self.total(trial_layout) saving = total - trial_total if round(saving, 2) <= 0: @@ -2066,6 +2086,16 @@ def _pricing( demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} costs.update((identity, hourly * (floor + value)) for identity, value in priced.items()) + above = sum(priced.values()) + rates = { + identity: receipt.throughput.requests_per_second + for identity in priced + if (receipt := receipts.get(identity)) is not None + } + if lane.envelope.floor_min and len(rates) > 1: + above = self.window.scale * sum( + max(0, min(lane.max_containers, need) - lane.envelope.floor_min) for need in self.lane_needs(rates) + ) memory = _UNPACKABLE if len(hosted) > 1: memory = self.memory( @@ -2078,7 +2108,7 @@ def _pricing( name=lane.name, identities=hosted, hourly_cost=hourly, - monthly_cost=hourly * (floor + sum(priced.values())), + monthly_cost=hourly * (floor + above), floor_monthly_cost=hourly * lane.envelope.floor_min * hours, envelope_max_monthly_cost=hourly * lane.envelope.floor_max * hours, unpriced_identities=tuple(identity for identity in hosted if identity not in priced), diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index 2129301cd..e6c4f3084 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -408,6 +408,33 @@ def test_two_never_cold_identities_share_a_card_on_their_co_residency_receipt(): assert again.report.identity("org/a").chosen.co_residents == ("org/b",) +BUSY = {"org/a": {0: 20000}, "org/b": {0: 20000}} + + +def test_a_shared_lane_prices_its_residents_demand_together(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), hourly=BUSY) + assert placements(result) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + (lane,) = result.report.lanes + assert lane.monthly_cost == pytest.approx(MONTH + MONTH / 24) + assert result.report.monthly_cost == pytest.approx(MONTH + MONTH / 24) + + +def test_a_packing_whose_residents_together_need_more_containers_than_the_ceiling_is_not_made(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), hourly=BUSY, max_containers=1) + assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert result.report.monthly_cost == pytest.approx(2 * MONTH) + assert not any( + Measurement.CO_RESIDENCY_RECEIPT in request.measure for request in result.report.measurement_requests + ) + quiet = run( + levels(**{"org/a": 0, "org/b": 0}), + pair_receipts(), + hourly={"org/a": {0: 10000}, "org/b": {0: 10000}}, + max_containers=1, + ) + assert placements(quiet) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + + def test_a_co_residency_receipt_over_a_warm_bound_keeps_the_identities_apart(): result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(b_p99=300.0)) assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} From 469bd429edb85b9d3c17d9197e3be1cb54a30ca2 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:16:24 +0100 Subject: [PATCH 33/40] docs(helm): say a stale admin token blocks only the applier's pool writes - Pool reads use the client token and pool writes the admin token, so a stale admin token refuses only the writes, while a stale client token refuses the reads and so the writes of the ticks that cannot read. - A failed connection is reported with the SDK's connection error message, any other transport failure by its exception type. --- deploy/helm/sie-cluster/README.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index c4e7abc50..d09e113e0 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -601,10 +601,11 @@ read-only as a file of mode `0440` owned by group 65532, under `--api-key-file`. The applier sends the admin token only on pool writes to the in-cluster gateway Service, never the client token on writes, and never takes a token on its command line or prints or logs one: a rejected token file is named -by its path, and a failed gateway call by the server's error message or the -exception type. Anyone who can read the applier's log sees lane telemetry and -floors, not credentials. Anyone who can create pods in the namespace with that -Secret can still read it, so the Secret read limits under +by its path, and a failed gateway call by the server's error message, a failed +connection by the SDK's connection error message, and any other transport +failure by its exception type. Anyone who can read the applier's log sees lane +telemetry and floors, not credentials. Anyone who can create pods in the +namespace with that Secret can still read it, so the Secret read limits under [Who holds which token](#who-holds-which-token) apply. The applier reads both token files once, when it starts. After you rotate the @@ -618,11 +619,13 @@ kubectl rollout restart deployment/-gateway -n kubectl rollout restart deployment/-placement-applier -n ``` -The cutover is complete once both run with the new token. Until then the -gateway refuses the applier's pool reads or writes and the pod keeps running: -each tick's audit rows record the failures, and in apply mode each carrier -lapses 1200 seconds after its last write and the lane returns to the largest -floor its chart and its other pools set. +The cutover is complete once both run with the new token. Pool reads use the +client token and pool writes the admin token, so a stale client token makes the +gateway refuse the applier's reads, and a tick that cannot read a carrier does +not write it, while a stale admin token makes it refuse only the writes. Either +way the pod keeps running: each tick's audit rows record the failures, and in +apply mode each carrier lapses 1200 seconds after its last write and the lane +returns to the largest floor its chart and its other pools set. The Deployment runs one replica with the `Recreate` strategy, so at most one applier writes carriers, also during a rollout. Its pod runs as user and group From 028fd4b56ce39c0fa7709db37ec9770aa2e8d188 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:51:44 +0100 Subject: [PATCH 34/40] fix(helm): compare credential Secret keys as the manifests name them - The token-separation check compared Secret references built from the raw values, while the manifests render most names and keys as unquoted YAML, which drops surrounding whitespace and anything after " #". A placement applier client token or MCP edge cluster API key of "admin " or "gateway-admin " passed the check and mounted the gateway admin token. - Every chart-rendered credential reference in sie-cluster.wiredSecretKeys now goes through sie-cluster.secretKeyRef, which trims the name and key and requires a Kubernetes Secret name and data key, so the reference and the manifest always name the same key. References that subcharts render are unchanged. - The applier and the MCP edge render their client Secret name and key trimmed and quoted. Default rendering is unchanged. --- deploy/helm/sie-cluster/README.md | 4 +- .../helm/sie-cluster/templates/_helpers.tpl | 61 +++++++++---- .../templates/mcp-edge-deployment.yaml | 6 +- .../templates/placement-applier.yaml | 18 ++-- tools/ci/tests/test_helm_render.py | 91 +++++++++++++++++++ 5 files changed, 148 insertions(+), 32 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index d09e113e0..888f5aaae 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -571,7 +571,9 @@ The render fails unless: holds one gateway client token, which the applier presents on pool reads. The applier presents the whole value, so the key must not list several comma-separated tokens, as an operator-managed `gateway.auth.tokenSecretName` - does during a [rotation](#gateway-token). + does during a [rotation](#gateway-token). The Secret name and key, after + trimming surrounding whitespace, are a Kubernetes Secret name and data key + that hold no other credential the chart wires. - With `placementApplier.apply: true` and gateway token auth, or a mode known only at run time, `gateway.auth.adminTokenSecretName` is set. - `gateway.extraEnv` sets `SIE_GATEWAY_POOL_MAX_MINIMUM_WORKER_COUNT` and diff --git a/deploy/helm/sie-cluster/templates/_helpers.tpl b/deploy/helm/sie-cluster/templates/_helpers.tpl index f67f372da..e827e62ab 100644 --- a/deploy/helm/sie-cluster/templates/_helpers.tpl +++ b/deploy/helm/sie-cluster/templates/_helpers.tpl @@ -1042,7 +1042,7 @@ every tool call may be refused. {{- $mcp := .Values.mcpEdge | default dict -}} {{- $auth := include "sie-cluster.gateway.effectiveAuthMode" . | fromJson -}} {{- $mayAuthenticate := or (include "sie-cluster.gateway.authenticates" .) (not $auth.known) -}} -{{- if and $mcp.enabled (not $mcp.clusterBaseUrl) (not (dig "clusterApiKey" "existingSecretName" "" $mcp)) $mayAuthenticate -}} +{{- if and $mcp.enabled (not $mcp.clusterBaseUrl) (not (trim (toString (dig "clusterApiKey" "existingSecretName" "" $mcp)))) $mayAuthenticate -}} {{- $example := "" -}} {{- with include "sie-cluster.gateway.tokenSecretName" . -}} {{- $example = printf ", for example mcpEdge.clusterApiKey.existingSecretName=%s and mcpEdge.clusterApiKey.secretKey=%s" . (include "sie-cluster.gateway.tokenSecretKey" $) -}} @@ -1085,12 +1085,13 @@ lanes and their envelopes. {{- fail "placementApplier.image.repository and placementApplier.image.tag are required: the chart publishes no image that runs sie-placement. Use an image that installs sie-placement with its keda extra." -}} {{- end -}} {{- $clientToken := $applier.clientToken | default dict -}} -{{- if and $clientToken.existingSecretName (not (trim (toString (default "" $clientToken.secretKey)))) -}} +{{- $clientSecret := trim (toString (default "" $clientToken.existingSecretName)) -}} +{{- if and $clientSecret (not (trim (toString (default "" $clientToken.secretKey)))) -}} {{- fail "placementApplier.clientToken.secretKey is empty. Set it to the key of placementApplier.clientToken.existingSecretName that holds the gateway client token." -}} {{- end -}} {{- $auth := include "sie-cluster.gateway.effectiveAuthMode" $root | fromJson -}} {{- $mayAuthenticate := or (include "sie-cluster.gateway.authenticates" $root) (not $auth.known) -}} -{{- if and $mayAuthenticate (not $clientToken.existingSecretName) -}} +{{- if and $mayAuthenticate (not $clientSecret) -}} {{- $example := "" -}} {{- if not (dig "auth" "tokenSecretName" "" ($root.Values.gateway | default dict)) -}} {{- with include "sie-cluster.gateway.tokenSecretName" $root -}} @@ -1207,6 +1208,24 @@ model code. Gateway client credentials may share a key with each other. {{- end -}} {{- end }} +{{/* +A credential reference "name/key" for sie-cluster.wiredSecretKeys. The name +and key are trimmed, as YAML reads an unquoted manifest field, and must then be +a Kubernetes Secret name and data key, so no spelling of either value in a +manifest names another Secret key than the reference does. +*/}} +{{- define "sie-cluster.secretKeyRef" -}} +{{- $name := trim (toString (default "" .name)) -}} +{{- $key := trim (toString (default "" .key)) -}} +{{- if not (and (le (len $name) 253) (regexMatch "^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$" $name)) -}} +{{- fail (printf "%s must name a Kubernetes Secret (lowercase letters, digits, '-' and '.'), got %q" .nameField $name) -}} +{{- end -}} +{{- if not (and (le (len $key) 253) (regexMatch "^[-._a-zA-Z0-9]+$" $key)) -}} +{{- fail (printf "%s must be a Kubernetes Secret data key (letters, digits, '-', '_' and '.'), got %q" .keyField $key) -}} +{{- end -}} +{{- printf "%s/%s" $name $key -}} +{{- end }} + {{/* Known chart credential keys, as a JSON list of {owner, name, ref, role}: owner is the values path or a description used in @@ -1220,47 +1239,47 @@ token-separation check and the upstream-key collision check read this list. {{- $keys := list -}} {{- $hfCache := $root.Values.workers.common.hfCache | default dict -}} {{- with $hfCache.tokenSecret -}} -{{- $keys = append $keys (dict "owner" "workers.common.hfCache.tokenSecret" "name" "Hugging Face token" "ref" (printf "%s/%s" . $hfCache.tokenSecretKey)) -}} +{{- $keys = append $keys (dict "owner" "workers.common.hfCache.tokenSecret" "name" "Hugging Face token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" $hfCache.tokenSecretKey "nameField" "workers.common.hfCache.tokenSecret" "keyField" "workers.common.hfCache.tokenSecretKey"))) -}} {{- end -}} {{- $gatewayAuth := $root.Values.gateway.auth | default dict -}} {{- with include "sie-cluster.gateway.tokenSecretName" $root -}} -{{- $keys = append $keys (dict "owner" (ternary "gateway.auth.tokenSecretName" "the generated gateway client token" (ne (dig "tokenSecretName" "" $gatewayAuth) "")) "name" "gateway client token" "ref" (printf "%s/%s" . (include "sie-cluster.gateway.tokenSecretKey" $root)) "role" "gateway-client") -}} +{{- $keys = append $keys (dict "owner" (ternary "gateway.auth.tokenSecretName" "the generated gateway client token" (ne (dig "tokenSecretName" "" $gatewayAuth) "")) "name" "gateway client token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" (include "sie-cluster.gateway.tokenSecretKey" $root) "nameField" "gateway.auth.tokenSecretName" "keyField" "gateway.auth.tokenSecretKey")) "role" "gateway-client") -}} {{- end -}} {{- range $entry := (default (list) $root.Values.gateway.extraEnv) -}} {{- if and (kindIs "map" $entry) (has (toString (index $entry "name")) (list "SIE_AUTH_TOKEN" "SIE_AUTH_TOKENS")) -}} {{- $secretRef := dig "valueFrom" "secretKeyRef" dict $entry -}} {{- if and $secretRef.name $secretRef.key -}} -{{- $keys = append $keys (dict "owner" (printf "gateway.extraEnv %s" (index $entry "name")) "name" (printf "gateway client token from gateway.extraEnv %s" (index $entry "name")) "ref" (printf "%s/%s" $secretRef.name $secretRef.key) "role" "gateway-client") -}} +{{- $keys = append $keys (dict "owner" (printf "gateway.extraEnv %s" (index $entry "name")) "name" (printf "gateway client token from gateway.extraEnv %s" (index $entry "name")) "ref" (include "sie-cluster.secretKeyRef" (dict "name" $secretRef.name "key" $secretRef.key "nameField" (printf "gateway.extraEnv %s valueFrom.secretKeyRef.name" (index $entry "name")) "keyField" (printf "gateway.extraEnv %s valueFrom.secretKeyRef.key" (index $entry "name")))) "role" "gateway-client") -}} {{- end -}} {{- end -}} {{- end -}} {{- with include "sie-cluster.workers.authTokenSecretName" $root -}} -{{- $keys = append $keys (dict "owner" (ternary "workers.auth.tokenSecretName" "the generated worker token" (ne (dig "auth" "tokenSecretName" "" $root.Values.workers) "")) "name" "worker token" "ref" (printf "%s/%s" . (include "sie-cluster.workers.authTokenSecretKey" $root)) "role" "worker") -}} +{{- $keys = append $keys (dict "owner" (ternary "workers.auth.tokenSecretName" "the generated worker token" (ne (dig "auth" "tokenSecretName" "" $root.Values.workers) "")) "name" "worker token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" (include "sie-cluster.workers.authTokenSecretKey" $root) "nameField" "workers.auth.tokenSecretName" "keyField" "workers.auth.tokenSecretKey")) "role" "worker") -}} {{- end -}} {{- with $gatewayAuth.adminTokenSecretName -}} -{{- $keys = append $keys (dict "owner" "gateway.auth.adminTokenSecretName" "name" "gateway admin token" "ref" (printf "%s/%s" . (include "sie-cluster.gateway.adminTokenSecretKey" $root))) -}} +{{- $keys = append $keys (dict "owner" "gateway.auth.adminTokenSecretName" "name" "gateway admin token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" (include "sie-cluster.gateway.adminTokenSecretKey" $root) "nameField" "gateway.auth.adminTokenSecretName" "keyField" "gateway.auth.adminTokenSecretKey"))) -}} {{- end -}} {{- with include "sie-cluster.config.readTokenSecretName" $root -}} -{{- $keys = append $keys (dict "owner" "the sie-config read token" "name" "sie-config read token" "ref" (printf "%s/%s" . (include "sie-cluster.config.readTokenSecretKey" $root))) -}} +{{- $keys = append $keys (dict "owner" "the sie-config read token" "name" "sie-config read token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" (include "sie-cluster.config.readTokenSecretKey" $root) "nameField" "config.auth.readTokenSecretName" "keyField" "config.auth.readTokenSecretKey"))) -}} {{- end -}} {{- with include "sie-cluster.config.adminTokenSecretName" $root -}} -{{- $keys = append $keys (dict "owner" "the sie-config admin token" "name" "sie-config admin token" "ref" (printf "%s/%s" . (include "sie-cluster.config.adminTokenSecretKey" $root))) -}} +{{- $keys = append $keys (dict "owner" "the sie-config admin token" "name" "sie-config admin token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" (include "sie-cluster.config.adminTokenSecretKey" $root) "nameField" "config.auth.adminTokenSecretName" "keyField" "config.auth.adminTokenSecretKey"))) -}} {{- end -}} {{- range $secretName := $root.Values.global.imagePullSecrets -}} {{- range $key := list ".dockerconfigjson" ".dockercfg" -}} -{{- $keys = append $keys (dict "owner" "global.imagePullSecrets" "name" "image pull secret" "ref" (printf "%s/%s" $secretName $key)) -}} +{{- $keys = append $keys (dict "owner" "global.imagePullSecrets" "name" "image pull secret" "ref" (include "sie-cluster.secretKeyRef" (dict "name" $secretName "key" $key "nameField" "global.imagePullSecrets" "keyField" "global.imagePullSecrets"))) -}} {{- end -}} {{- end -}} {{- $oauthSecret := $root.Values.auth.oauth2Proxy.secret -}} {{- with $oauthSecret.name -}} {{- $secretName := . -}} {{- range $keyField := list "clientIDKey" "clientSecretKey" "cookieSecretKey" -}} -{{- $keys = append $keys (dict "owner" (printf "auth.oauth2Proxy.secret.%s" $keyField) "name" (printf "oauth2-proxy %s" $keyField) "ref" (printf "%s/%s" $secretName (index $oauthSecret $keyField))) -}} +{{- $keys = append $keys (dict "owner" (printf "auth.oauth2Proxy.secret.%s" $keyField) "name" (printf "oauth2-proxy %s" $keyField) "ref" (include "sie-cluster.secretKeyRef" (dict "name" $secretName "key" (index $oauthSecret $keyField) "nameField" "auth.oauth2Proxy.secret.name" "keyField" (printf "auth.oauth2Proxy.secret.%s" $keyField)))) -}} {{- end -}} {{- end -}} {{- $betterStack := $root.Values.observability.otel.collector.betterStack -}} {{- with $betterStack.existingSecret -}} -{{- $keys = append $keys (dict "owner" "observability.otel.collector.betterStack.existingSecret" "name" "Better Stack token" "ref" (printf "%s/%s" . $betterStack.tokenKey)) -}} +{{- $keys = append $keys (dict "owner" "observability.otel.collector.betterStack.existingSecret" "name" "Better Stack token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" $betterStack.tokenKey "nameField" "observability.otel.collector.betterStack.existingSecret" "keyField" "observability.otel.collector.betterStack.tokenKey"))) -}} {{- end -}} {{- with index $root.Subcharts "kube-prometheus-stack" -}} {{- with index .Subcharts "grafana" -}} @@ -1352,15 +1371,17 @@ token-separation check and the upstream-key collision check read this list. {{- end -}} {{- $mcp := $root.Values.mcpEdge -}} {{- if or $mcp.connectorSecrets $mcp.existingSecretName -}} -{{- $keys = append $keys (dict "owner" "the MCP connector secrets" "name" "MCP connector secrets" "ref" (printf "%s/connector-secrets" (include "sie-cluster.mcpEdge.secretName" $root))) -}} +{{- $keys = append $keys (dict "owner" "the MCP connector secrets" "name" "MCP connector secrets" "ref" (include "sie-cluster.secretKeyRef" (dict "name" (include "sie-cluster.mcpEdge.secretName" $root) "key" "connector-secrets" "nameField" "mcpEdge.existingSecretName" "keyField" "mcpEdge.existingSecretName"))) -}} {{- end -}} -{{- with $mcp.clusterApiKey.existingSecretName -}} -{{- $keys = append $keys (dict "owner" "mcpEdge.clusterApiKey.existingSecretName" "name" "MCP edge cluster API key" "ref" (printf "%s/%s" . $mcp.clusterApiKey.secretKey) "role" "gateway-client") -}} +{{- with trim (toString (default "" $mcp.clusterApiKey.existingSecretName)) -}} +{{- $keys = append $keys (dict "owner" "mcpEdge.clusterApiKey.existingSecretName" "name" "MCP edge cluster API key" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" $mcp.clusterApiKey.secretKey "nameField" "mcpEdge.clusterApiKey.existingSecretName" "keyField" "mcpEdge.clusterApiKey.secretKey")) "role" "gateway-client") -}} {{- end -}} {{- if include "sie-cluster.placementApplier.enabled" $root -}} {{- $applierToken := dig "clientToken" dict ($root.Values.placementApplier | default dict) -}} -{{- with $applierToken.existingSecretName -}} -{{- $keys = append $keys (dict "owner" "placementApplier.clientToken.existingSecretName" "name" "placement applier client token" "ref" (printf "%s/%s" . $applierToken.secretKey) "role" "gateway-client") -}} +{{- with trim (toString (default "" $applierToken.existingSecretName)) -}} +{{- if trim (toString (default "" $applierToken.secretKey)) -}} +{{- $keys = append $keys (dict "owner" "placementApplier.clientToken.existingSecretName" "name" "placement applier client token" "ref" (include "sie-cluster.secretKeyRef" (dict "name" . "key" $applierToken.secretKey "nameField" "placementApplier.clientToken.existingSecretName" "keyField" "placementApplier.clientToken.secretKey")) "role" "gateway-client") -}} +{{- end -}} {{- end -}} {{- end -}} {{- $tls := include "sie-cluster.ingressTlsConfig" $root | fromYaml -}} @@ -1373,7 +1394,7 @@ token-separation check and the upstream-key collision check read this list. {{- range $tlsSecret := $tlsSecrets -}} {{- if $tlsSecret.name -}} {{- range $key := list "tls.crt" "tls.key" -}} -{{- $keys = append $keys (dict "owner" $tlsSecret.owner "name" (trimPrefix "the " $tlsSecret.owner) "ref" (printf "%s/%s" $tlsSecret.name $key)) -}} +{{- $keys = append $keys (dict "owner" $tlsSecret.owner "name" (trimPrefix "the " $tlsSecret.owner) "ref" (include "sie-cluster.secretKeyRef" (dict "name" $tlsSecret.name "key" $key "nameField" $tlsSecret.owner "keyField" $tlsSecret.owner))) -}} {{- end -}} {{- end -}} {{- end -}} @@ -1384,7 +1405,7 @@ token-separation check and the upstream-key collision check read this list. {{- end -}} {{- range $component := $components -}} {{- $natsSecret := include "sie-cluster.nats.authSecretName" (dict "auth" $root.Values.nats.auth "release" $root.Release.Name "component" $component) -}} -{{- $keys = append $keys (dict "owner" (printf "the NATS %s password" $component) "name" (printf "NATS %s password" $component) "ref" (printf "%s/password" $natsSecret)) -}} +{{- $keys = append $keys (dict "owner" (printf "the NATS %s password" $component) "name" (printf "NATS %s password" $component) "ref" (include "sie-cluster.secretKeyRef" (dict "name" $natsSecret "key" "password" "nameField" (printf "nats.auth.existingSecrets.%s" $component) "keyField" (printf "nats.auth.existingSecrets.%s" $component)))) -}} {{- end -}} {{- end -}} {{- $keys | toJson -}} diff --git a/deploy/helm/sie-cluster/templates/mcp-edge-deployment.yaml b/deploy/helm/sie-cluster/templates/mcp-edge-deployment.yaml index ce76a8502..ed503ae3d 100644 --- a/deploy/helm/sie-cluster/templates/mcp-edge-deployment.yaml +++ b/deploy/helm/sie-cluster/templates/mcp-edge-deployment.yaml @@ -58,12 +58,12 @@ spec: - name: SIE_MCP_OAUTH_ENABLED value: "0" {{- end }} - {{- if .Values.mcpEdge.clusterApiKey.existingSecretName }} + {{- with trim (toString (default "" .Values.mcpEdge.clusterApiKey.existingSecretName)) }} - name: SIE_API_KEY valueFrom: secretKeyRef: - name: {{ .Values.mcpEdge.clusterApiKey.existingSecretName }} - key: {{ .Values.mcpEdge.clusterApiKey.secretKey }} + name: {{ . | quote }} + key: {{ trim (toString (default "" $.Values.mcpEdge.clusterApiKey.secretKey)) | quote }} {{- end }} - name: SIE_LOG_LEVEL value: {{ .Values.logging.level | quote }} diff --git a/deploy/helm/sie-cluster/templates/placement-applier.yaml b/deploy/helm/sie-cluster/templates/placement-applier.yaml index 357a068cd..63279f7d3 100644 --- a/deploy/helm/sie-cluster/templates/placement-applier.yaml +++ b/deploy/helm/sie-cluster/templates/placement-applier.yaml @@ -26,9 +26,11 @@ {{- end -}} {{- end -}} {{- $clientToken := $applier.clientToken | default dict -}} +{{- $clientSecret := trim (toString (default "" $clientToken.existingSecretName)) -}} +{{- $clientKey := trim (toString (default "" $clientToken.secretKey)) -}} {{- $adminTokenSecret := "" -}} {{- if $apply -}} -{{- $adminTokenSecret = dig "auth" "adminTokenSecretName" "" (.Values.gateway | default dict) -}} +{{- $adminTokenSecret = dig "auth" "adminTokenSecretName" "" (.Values.gateway | default dict) | toString | trim -}} {{- end -}} apiVersion: v1 kind: ServiceAccount @@ -110,7 +112,7 @@ spec: - {{ .flag }} - {{ .value | quote }} {{- end }} - {{- if $clientToken.existingSecretName }} + {{- if $clientSecret }} - --api-key-file - /etc/sie-placement/client-token/token {{- end }} @@ -132,7 +134,7 @@ spec: - name: plan mountPath: /etc/sie-placement/plan readOnly: true - {{- if $clientToken.existingSecretName }} + {{- if $clientSecret }} - name: client-token mountPath: /etc/sie-placement/client-token readOnly: true @@ -149,22 +151,22 @@ spec: items: - key: plan.json path: plan.json - {{- with $clientToken.existingSecretName }} + {{- with $clientSecret }} - name: client-token secret: - secretName: {{ . }} + secretName: {{ . | quote }} defaultMode: 0440 items: - - key: {{ $clientToken.secretKey }} + - key: {{ $clientKey | quote }} path: token {{- end }} {{- with $adminTokenSecret }} - name: admin-token secret: - secretName: {{ . }} + secretName: {{ . | quote }} defaultMode: 0440 items: - - key: {{ include "sie-cluster.gateway.adminTokenSecretKey" $ }} + - key: {{ include "sie-cluster.gateway.adminTokenSecretKey" $ | quote }} path: token {{- end }} {{- with $applier.nodeSelector }} diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 9879c9667..3f3a0afe8 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -3586,6 +3586,42 @@ def test_the_mcp_edge_renders_when_it_has_what_its_gateway_needs(tmp_path: Path, rendered_documents(tmp_path, {**L4_POOL, **values}) +MCP_ADMIN_AUTH = { + "gateway": { + "auth": { + "mode": "token", + "generateToken": True, + "adminTokenSecretName": "gateway-admin", + "adminTokenSecretKey": "admin", + } + } +} + + +@pytest.mark.parametrize( + "api_key", + [ + {"existingSecretName": "gateway-admin", "secretKey": "admin "}, + {"existingSecretName": "gateway-admin ", "secretKey": "admin"}, + ], + ids=["key-space", "name-space"], +) +def test_the_mcp_edge_api_key_is_kept_apart_from_the_admin_token_whatever_its_spacing( + tmp_path: Path, api_key: dict +) -> None: + values = {**L4_POOL, **MCP_ADMIN_AUTH, "mcpEdge": {"enabled": True, "clusterApiKey": api_key}} + assert ( + "The MCP edge cluster API key and the gateway admin token are the same Secret key (gateway-admin/admin)" + in render_error(tmp_path, values) + ) + + +def test_the_mcp_edge_reads_the_trimmed_api_key_secret_key(tmp_path: Path) -> None: + api_key = {"existingSecretName": " mcp-key ", "secretKey": " token "} + docs = rendered_documents(tmp_path, {**L4_POOL, **MCP_TOKEN_AUTH, "mcpEdge": {"enabled": True, "clusterApiKey": api_key}}) + assert env_entries(docs, "SIE_API_KEY") == {"sie-sie-cluster-mcp/mcp-edge": {"name": "mcp-key", "key": "token"}} + + def test_a_gateway_mode_from_value_from_gets_the_generated_token(tmp_path: Path) -> None: mode = {"name": "SIE_AUTH_MODE", "valueFrom": {"configMapKeyRef": {"name": "gateway-auth", "key": "mode"}}} docs = rendered_documents(tmp_path, {**L4_POOL, "gateway": {"auth": {"generateToken": True}, "extraEnv": [mode]}}) @@ -4605,6 +4641,61 @@ def test_the_applier_client_token_is_kept_apart_from_the_admin_token(tmp_path: P ) +@pytest.mark.parametrize( + "client", + [ + {"existingSecretName": "gateway-admin", "secretKey": "admin "}, + {"existingSecretName": "gateway-admin ", "secretKey": "admin"}, + {"existingSecretName": " gateway-admin", "secretKey": "\tadmin"}, + ], + ids=["key-space", "name-space", "leading-space"], +) +@pytest.mark.parametrize("apply", [False, True]) +def test_the_applier_client_token_is_kept_apart_from_the_admin_token_whatever_its_spacing( + tmp_path: Path, client: dict, apply: bool +) -> None: + result = render_placement(tmp_path, *applier_layers(APPLIER_AUTH, apply=apply, clientToken=client)) + assert result.returncode != 0 + assert ( + "The placement applier client token and the gateway admin token are the same Secret key (gateway-admin/admin)" + in result.stderr + ) + + +@pytest.mark.parametrize( + ("client", "error"), + [ + ( + {"existingSecretName": "gateway-admin #x", "secretKey": "admin"}, + 'placementApplier.clientToken.existingSecretName must name a Kubernetes Secret (lowercase letters, digits, ' + "'-' and '.'), got \"gateway-admin #x\"", + ), + ( + {"existingSecretName": "applier-client", "secretKey": "admin #x"}, + "placementApplier.clientToken.secretKey must be a Kubernetes Secret data key (letters, digits, '-', '_' " + "and '.'), got \"admin #x\"", + ), + ], +) +def test_an_applier_client_token_that_is_not_a_secret_key_fails_the_render( + tmp_path: Path, client: dict, error: str +) -> None: + result = render_placement(tmp_path, *applier_layers(APPLIER_AUTH, clientToken=client)) + assert result.returncode != 0 + assert error in result.stderr + + +def test_the_applier_mounts_the_trimmed_client_token_secret_key(tmp_path: Path) -> None: + client = {"existingSecretName": " applier-client ", "secretKey": " token "} + spec = applier_pod(placement_documents(tmp_path, applier_layers(APPLIER_AUTH, clientToken=client)))["spec"] + volumes = {volume["name"]: volume for volume in spec["volumes"]} + assert volumes["client-token"]["secret"] == { + "secretName": "applier-client", + "defaultMode": SECRET_FILE_MODE, + "items": [{"key": "token", "path": "token"}], + } + + def envelope_values(settings: dict, *, autoscaling: bool = False) -> dict: values = {"workers": {"pools": {"l4": {"enabled": True, "bundles": {"default": settings}}}}} return {**values, **AUTOSCALING_VALUES} if autoscaling else values From d3adf0b71da68aeddd2c67f2cfacfb16f8ced258 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:52:12 +0100 Subject: [PATCH 35/40] fix(helm): accept only Kubernetes pull policies for the placement applier - placementApplier.image.pullPolicy must be Always, IfNotPresent or Never, and renders quoted, so a value carrying YAML cannot add fields such as an env list to the applier container. Its Secret names and keys already render quoted. --- deploy/helm/sie-cluster/README.md | 1 + .../helm/sie-cluster/templates/_helpers.tpl | 4 ++++ .../templates/placement-applier.yaml | 2 +- tools/ci/tests/test_helm_render.py | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 888f5aaae..c6f05f1ce 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -558,6 +558,7 @@ The render fails unless: - `placementApplier.image.repository` and `placementApplier.image.tag` name an image that installs `sie-placement` with its `keda` extra and can run as user and group 65532. The chart publishes no such image. + `placementApplier.image.pullPolicy` is `Always`, `IfNotPresent` or `Never`. - `placementApplier.plan` is a placement-plan document. Each of its lanes names an enabled worker lane by its `queuePool`, `cell.machineProfile` and `bundle`, compared as the chart canonicalizes lane tokens, and that worker diff --git a/deploy/helm/sie-cluster/templates/_helpers.tpl b/deploy/helm/sie-cluster/templates/_helpers.tpl index e827e62ab..541256295 100644 --- a/deploy/helm/sie-cluster/templates/_helpers.tpl +++ b/deploy/helm/sie-cluster/templates/_helpers.tpl @@ -1084,6 +1084,10 @@ lanes and their envelopes. {{- if not (and (trim (toString (default "" $image.repository))) (trim (toString (default "" $image.tag)))) -}} {{- fail "placementApplier.image.repository and placementApplier.image.tag are required: the chart publishes no image that runs sie-placement. Use an image that installs sie-placement with its keda extra." -}} {{- end -}} +{{- $pullPolicy := toString (default "IfNotPresent" $image.pullPolicy) -}} +{{- if not (has $pullPolicy (list "Always" "IfNotPresent" "Never")) -}} +{{- fail (printf "placementApplier.image.pullPolicy must be Always, IfNotPresent or Never, got %q" $pullPolicy) -}} +{{- end -}} {{- $clientToken := $applier.clientToken | default dict -}} {{- $clientSecret := trim (toString (default "" $clientToken.existingSecretName)) -}} {{- if and $clientSecret (not (trim (toString (default "" $clientToken.secretKey)))) -}} diff --git a/deploy/helm/sie-cluster/templates/placement-applier.yaml b/deploy/helm/sie-cluster/templates/placement-applier.yaml index 63279f7d3..a859a14a6 100644 --- a/deploy/helm/sie-cluster/templates/placement-applier.yaml +++ b/deploy/helm/sie-cluster/templates/placement-applier.yaml @@ -96,7 +96,7 @@ spec: containers: - name: applier image: {{ printf "%s:%s" (trim (toString $applier.image.repository)) (trim (toString $applier.image.tag)) | quote }} - imagePullPolicy: {{ $applier.image.pullPolicy | default "IfNotPresent" }} + imagePullPolicy: {{ $applier.image.pullPolicy | default "IfNotPresent" | quote }} command: ["sie-placement"] args: - tune diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 3f3a0afe8..4c1d04568 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4696,6 +4696,25 @@ def test_the_applier_mounts_the_trimmed_client_token_secret_key(tmp_path: Path) } +INJECTED_PULL_POLICY = "IfNotPresent\n env:\n - name: INJECTED\n value: x" + + +@pytest.mark.parametrize("policy", ["Always", "IfNotPresent", "Never"]) +def test_the_applier_takes_a_kubernetes_pull_policy(tmp_path: Path, policy: str) -> None: + image = {**APPLIER_IMAGE, "pullPolicy": policy} + (container,) = applier_pod(placement_documents(tmp_path, applier_layers(image=image)))["spec"]["containers"] + assert container["imagePullPolicy"] == policy + assert "env" not in container + + +@pytest.mark.parametrize("policy", [INJECTED_PULL_POLICY, "ifnotpresent", "Sometimes"]) +def test_an_applier_pull_policy_kubernetes_does_not_know_fails_the_render(tmp_path: Path, policy: str) -> None: + result = render_placement(tmp_path, *applier_layers(image={**APPLIER_IMAGE, "pullPolicy": policy})) + assert result.returncode != 0 + assert "placementApplier.image.pullPolicy must be Always, IfNotPresent or Never" in result.stderr + assert "INJECTED" not in result.stdout + + def envelope_values(settings: dict, *, autoscaling: bool = False) -> dict: values = {"workers": {"pools": {"l4": {"enabled": True, "bundles": {"default": settings}}}}} return {**values, **AUTOSCALING_VALUES} if autoscaling else values From 6f83ed818eb6cdd88d99b1231d35168029bd54fd Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:52:18 +0100 Subject: [PATCH 36/40] fix(helm): mount the applier's admin token only when the gateway may check it - With placementApplier.apply true, the chart now mounts the gateway admin token and passes --admin-token-file only when the gateway enforces token auth or its auth mode is known only at run time, the same condition under which the render requires gateway.auth.adminTokenSecretName. A gateway whose auth mode is none checks no token, so the applier no longer holds a credential it does not need there. --- deploy/helm/sie-cluster/README.md | 2 +- deploy/helm/sie-cluster/templates/placement-applier.yaml | 4 +++- tools/ci/tests/test_helm_render.py | 9 +++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index c6f05f1ce..7fb222917 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -594,7 +594,7 @@ not read as `null`. The audit row's `error` records a failed pool call. | Credential | Held | Authorizes | | --- | --- | --- | -| Gateway admin token | Only with `placementApplier.apply: true`, from the Secret `gateway.auth.adminTokenSecretName` (key `gateway.auth.adminTokenSecretKey`) | `POST`, `PUT` and `DELETE` under `/v1/pools`, `/v1/admin` and `/v1/configs`: every gateway admin route, not only floor carriers | +| Gateway admin token | Only with `placementApplier.apply: true` and a gateway that may authenticate, with token auth or a mode known only at run time, from the Secret `gateway.auth.adminTokenSecretName` (key `gateway.auth.adminTokenSecretKey`) | `POST`, `PUT` and `DELETE` under `/v1/pools`, `/v1/admin` and `/v1/configs`: every gateway admin route, not only floor carriers | | Gateway client token | When `placementApplier.clientToken.existingSecretName` is set | Every gateway route that is not an admin route, inference included; the applier uses it for `GET /v1/pools/{name}` | | Kubernetes API | Never | Nothing: the applier has its own ServiceAccount, mounts no ServiceAccount token, and has no Role or RoleBinding | diff --git a/deploy/helm/sie-cluster/templates/placement-applier.yaml b/deploy/helm/sie-cluster/templates/placement-applier.yaml index a859a14a6..07b63f731 100644 --- a/deploy/helm/sie-cluster/templates/placement-applier.yaml +++ b/deploy/helm/sie-cluster/templates/placement-applier.yaml @@ -28,8 +28,10 @@ {{- $clientToken := $applier.clientToken | default dict -}} {{- $clientSecret := trim (toString (default "" $clientToken.existingSecretName)) -}} {{- $clientKey := trim (toString (default "" $clientToken.secretKey)) -}} +{{- $auth := include "sie-cluster.gateway.effectiveAuthMode" . | fromJson -}} +{{- $mayAuthenticate := or (include "sie-cluster.gateway.authenticates" .) (not $auth.known) -}} {{- $adminTokenSecret := "" -}} -{{- if $apply -}} +{{- if and $apply $mayAuthenticate -}} {{- $adminTokenSecret = dig "auth" "adminTokenSecretName" "" (.Values.gateway | default dict) | toString | trim -}} {{- end -}} apiVersion: v1 diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 4c1d04568..1619c4b28 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -4715,6 +4715,15 @@ def test_an_applier_pull_policy_kubernetes_does_not_know_fails_the_render(tmp_pa assert "INJECTED" not in result.stdout +def test_the_applier_holds_no_admin_token_when_the_gateway_authenticates_nothing(tmp_path: Path) -> None: + auth = {"gateway": {"auth": {"mode": "none", "adminTokenSecretName": "gateway-admin", "adminTokenSecretKey": "admin"}}} + spec = applier_pod(placement_documents(tmp_path, applier_layers(auth, apply=True)))["spec"] + (container,) = spec["containers"] + assert container["args"] == [*APPLIER_ARGS, "--apply"] + assert [mount["name"] for mount in container["volumeMounts"]] == ["plan"] + assert [volume["name"] for volume in spec["volumes"]] == ["plan"] + + def envelope_values(settings: dict, *, autoscaling: bool = False) -> dict: values = {"workers": {"pools": {"l4": {"enabled": True, "bundles": {"default": settings}}}}} return {**values, **AUTOSCALING_VALUES} if autoscaling else values From 70138e38ec30e82f8b8f9c50fd2bb60be4928c4c Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:59:22 +0100 Subject: [PATCH 37/40] docs(helm): say the applier reports a gateway error reply by its status alone --- deploy/helm/sie-cluster/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 7fb222917..0b37bd78d 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -604,12 +604,13 @@ read-only as a file of mode `0440` owned by group 65532, under `--api-key-file`. The applier sends the admin token only on pool writes to the in-cluster gateway Service, never the client token on writes, and never takes a token on its command line or prints or logs one: a rejected token file is named -by its path, and a failed gateway call by the server's error message, a failed -connection by the SDK's connection error message, and any other transport -failure by its exception type. Anyone who can read the applier's log sees lane -telemetry and floors, not credentials. Anyone who can create pods in the -namespace with that Secret can still read it, so the Secret read limits under -[Who holds which token](#who-holds-which-token) apply. +by its path, a gateway call that gets an error reply by its HTTP status alone, +so no text the server returns reaches the log, a failed connection by the SDK's +connection error message, and any other transport failure by its exception +type. Anyone who can read the applier's log sees lane telemetry and floors, not +credentials. Anyone who can create pods in the namespace with that Secret can +still read it, so the Secret read limits under [Who holds which +token](#who-holds-which-token) apply. The applier reads both token files once, when it starts. After you rotate the gateway client token, restart it as step 2 of From f860df67215505212558ffcd1834342e8661d92a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 12:11:39 +0100 Subject: [PATCH 38/40] docs(placement): pass the planner's plan to helm_values and scope enabled false - The helm_values example passes result.plan, the planner output, rather than the plan function. - Setting other bundles to enabled false covers only the worker groups the overlay emits; a worker group the plan does not use keeps the values beneath the overlay, so the lanes they enable still run. --- packages/sie_placement/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 0fdb28e95..0196ac3e4 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -537,8 +537,9 @@ The report renders as a `placement-report` document: ## Helm values -`helm_values` turns a plan into a values overlay for the `sie-cluster` Helm -chart, to pass with `-f` after the release's own values: +`helm_values` turns a plan, such as the `result.plan` of the planner example +above, into a values overlay for the `sie-cluster` Helm chart, to pass with +`-f` after the release's own values: ```python import json @@ -546,7 +547,7 @@ import json from sie_placement.helm import ChartParameters, helm_values with open("placement-values.json", "w", encoding="utf-8") as handle: - json.dump(helm_values(plan, ChartParameters(fallback_replicas=2)), handle, indent=2) + json.dump(helm_values(result.plan, ChartParameters(fallback_replicas=2)), handle, indent=2) ``` | Parameter | Meaning | @@ -567,8 +568,10 @@ overlay sets: `resources` requests and limits equal to the container shape. Each chart lane is an enabled `bundles.` entry whose `minReplicas` is the envelope's `floorMin` and whose `maxReplicas` is `maxContainers`. Every other - bundle that `base_pools` enables under the group is set to - `enabled: false`, so the release runs no lane the plan does not place. + bundle that `base_pools` enables under the group is set to `enabled: false`, + so a worker group the overlay emits runs no lane the plan does not place on + it. A worker group the plan does not use is not in the overlay and keeps the + values beneath it, so the lanes they enable there still run outside the plan. `gpu.deviceGroup` is `false` for a single GPU and the `device_groups` entry of the machine profile for more. - `queueRouting.staticQueuePools.`: one static queue pool for every From 14214ce3389e458dc7df0f7084e93564538a3708 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 12:12:28 +0100 Subject: [PATCH 39/40] docs(helm): say what the chart checks in the applier plan and what the applier does - The render needs placementApplier.plan to be a map of kind placement-plan with at least one lane, and checks only those fields and the lane envelopes. The applier parses the whole plan when it starts and refuses one that is not a valid placement-plan document, an unsupported schemaVersion included. --- deploy/helm/sie-cluster/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 0b37bd78d..727375694 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -559,14 +559,18 @@ The render fails unless: image that installs `sie-placement` with its `keda` extra and can run as user and group 65532. The chart publishes no such image. `placementApplier.image.pullPolicy` is `Always`, `IfNotPresent` or `Never`. -- `placementApplier.plan` is a placement-plan document. Each of its lanes names - an enabled worker lane by its `queuePool`, `cell.machineProfile` and - `bundle`, compared as the chart canonicalizes lane tokens, and that worker - lane declares an `envelope` equal to the plan lane's. A plan therefore cannot - move a floor outside a range that the chart values bound. The values overlay - that `helm_values` in `sie-placement` emits from a plan sets each lane's +- `placementApplier.plan` is a map whose `kind` is `placement-plan` and whose + `lanes` list holds at least one lane. Each of its lanes names an enabled + worker lane by its `queuePool`, `cell.machineProfile` and `bundle`, compared + as the chart canonicalizes lane tokens, and that worker lane declares an + `envelope` equal to the plan lane's. A plan therefore cannot move a floor + outside a range that the chart values bound. The values overlay that + `helm_values` in `sie-placement` emits from a plan sets each lane's `minReplicas` and `maxReplicas` but no `envelope`, so copy each plan lane's `envelope.floorMin` and `envelope.floorMax` into the worker lane it names. + The chart checks only these fields. The applier parses the whole plan when + it starts and refuses one that is not a valid placement-plan document, an + unsupported `schemaVersion` included. - With gateway token auth, or a gateway auth mode known only at run time, `placementApplier.clientToken.existingSecretName` names a Secret key that holds one gateway client token, which the applier presents on pool reads. From d5f8a5b2dec87a0562ddde9b389fe0ad1f5c326c Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 15:28:11 +0100 Subject: [PATCH 40/40] style(ci): format the Helm render tests --- tools/ci/tests/test_helm_render.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index 1619c4b28..5e083ad71 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -3618,7 +3618,9 @@ def test_the_mcp_edge_api_key_is_kept_apart_from_the_admin_token_whatever_its_sp def test_the_mcp_edge_reads_the_trimmed_api_key_secret_key(tmp_path: Path) -> None: api_key = {"existingSecretName": " mcp-key ", "secretKey": " token "} - docs = rendered_documents(tmp_path, {**L4_POOL, **MCP_TOKEN_AUTH, "mcpEdge": {"enabled": True, "clusterApiKey": api_key}}) + docs = rendered_documents( + tmp_path, {**L4_POOL, **MCP_TOKEN_AUTH, "mcpEdge": {"enabled": True, "clusterApiKey": api_key}} + ) assert env_entries(docs, "SIE_API_KEY") == {"sie-sie-cluster-mcp/mcp-edge": {"name": "mcp-key", "key": "token"}} @@ -4667,7 +4669,7 @@ def test_the_applier_client_token_is_kept_apart_from_the_admin_token_whatever_it [ ( {"existingSecretName": "gateway-admin #x", "secretKey": "admin"}, - 'placementApplier.clientToken.existingSecretName must name a Kubernetes Secret (lowercase letters, digits, ' + "placementApplier.clientToken.existingSecretName must name a Kubernetes Secret (lowercase letters, digits, " "'-' and '.'), got \"gateway-admin #x\"", ), ( @@ -4716,7 +4718,9 @@ def test_an_applier_pull_policy_kubernetes_does_not_know_fails_the_render(tmp_pa def test_the_applier_holds_no_admin_token_when_the_gateway_authenticates_nothing(tmp_path: Path) -> None: - auth = {"gateway": {"auth": {"mode": "none", "adminTokenSecretName": "gateway-admin", "adminTokenSecretKey": "admin"}}} + auth = { + "gateway": {"auth": {"mode": "none", "adminTokenSecretName": "gateway-admin", "adminTokenSecretKey": "admin"}} + } spec = applier_pod(placement_documents(tmp_path, applier_layers(auth, apply=True)))["spec"] (container,) = spec["containers"] assert container["args"] == [*APPLIER_ARGS, "--apply"]