From c936cadfe3e11d5b970b46e9843ac9fab63dc37e Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 17:58:19 +0100 Subject: [PATCH 01/10] feat(placement): pack several models onto one GPU and price shared lanes jointly One card can hold several models: some pinned and always resident, the rest loaded on demand. The planner now forms such lanes from measured evidence and prices a shared lane once for all its residents. - Receipts record footprintMiB, workingMemoryMiB and coResidents, boot records the footprintMiB of a load that succeeded, and price-list profiles gpuMemoryMiB. A lane fits when its pinned footprints, its largest lazy footprint and its largest working memory at the lane's concurrency come to at most 80% of the card. Only single-card lanes without fallback cells hold several identities. - A pinned identity beside other pinned identities is judged by a co-residency receipt whose coResidents include them, and its demand is priced at that receipt. A receipt measured beside more residents stands for fewer. A current shared lane without one is kept and asks for it. - Lazy-resident options on current host lanes need the memory check, and a footprint or working memory that is not on file is a measurement request. - A packing pass visits every identity alone on its lane in decreasing floor cost and tries lanes that pin a never-cold identity first, then in decreasing footprint in use. It keeps a move only when the plan total falls and every resident's service level holds, and a move that lacks footprints, a load or a co-residency receipt becomes a measurement request bounded by that fall. - Shared lanes are priced jointly: identities that left a lane their neighbours keep return to it, alone or together, whenever that does not raise the plan total, and an identity whose co-pinned neighbours change in the run is decided again beside the ones it is left with. Loosening a bound can no longer raise the plan total because a pinned identity left a shared lane; the levers test that pinned that defect now asserts the identity stays. - Generation identities are never packed. --- packages/sie_placement/README.md | 143 +++- .../src/sie_placement/planner.py | 590 ++++++++++++++- .../sie_placement/src/sie_placement/report.py | 10 + .../sie_placement/src/sie_placement/schema.py | 49 +- .../fixtures/planner/expected-report.json | 14 +- .../tests/fixtures/planner/price-list.json | 4 + .../tests/fixtures/planner/receipts.json | 44 ++ .../sie_placement/tests/test_availability.py | 11 +- packages/sie_placement/tests/test_levers.py | 20 +- packages/sie_placement/tests/test_packing.py | 694 ++++++++++++++++++ packages/sie_placement/tests/test_planner.py | 139 ++-- packages/sie_placement/tests/test_schema.py | 73 ++ 12 files changed, 1668 insertions(+), 123 deletions(-) create mode 100644 packages/sie_placement/tests/test_packing.py diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 304094fcf..494d2cfc7 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -25,9 +25,9 @@ name the offending field by its JSON path, for example | Kind | Type | Contents | | --- | --- | --- | | `placement-service-levels` | `ServiceLevels` | Optional named `tiers` and the `identities` that declare or inherit service levels | -| `placement-receipt` | `Receipt` | One throughput and latency measurement of one identity on one cell, container shape, concurrency and request shape | -| `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape | -| `placement-price-list` | `PriceList` | Accelerator, core and memory rates per machine profile, and the region multiplier per placement | +| `placement-receipt` | `Receipt` | One throughput and latency measurement of one identity on one cell, container shape, concurrency and request shape, alone or beside named co-residents, with the GPU memory the identity holds | +| `placement-boot-evidence` | `BootEvidence` | One measured boot, snapshot restore or resident load of one identity on one cell and container shape, with the GPU memory a successful one left in use | +| `placement-price-list` | `PriceList` | Accelerator, core and memory rates and the card memory per machine profile, and the region multiplier per placement | | `placement-demand` | `Demand` | Hourly request counts per identity and operation over a UTC window | | `placement-capacity` | `Capacity` | The accelerators a plan may hold: in total, per GPU product, and in a quota it shares with other environments | | `placement-plan` | `Plan` | Lanes with their cell, fallback cells, container shape, concurrency, ceiling and envelope, and the lane and warmth of every identity | @@ -120,6 +120,40 @@ A lane holds `floorMax` containers of its `gpuCount` accelerators each. A lane with fallback cells counts against the ceiling of every product it names, and once against `total`. +## Several models on one GPU + +A lane can hold several identities on one card: some `pinned`, always +resident, and the rest `lazy-resident`, loaded on demand and evicted least +recently used. The planner decides this from measurements only: + +- `footprintMiB`, on a receipt or on a boot record that succeeded, is the GPU + memory the identity holds once loaded: the memory in use after its load less + the memory in use before it. The planner takes the largest an identity + records on a cell. +- `workingMemoryMiB`, on a receipt, is the GPU memory its run needed at the + receipt's concurrency beyond the footprint. The planner takes the largest an + identity records at a cell, container shape and concurrency. +- `coResidents`, on a receipt, names the identities pinned and loaded on the + same container while it was measured, which makes it a co-residency receipt. + A receipt without `coResidents` measured the identity alone and is a cell; + a co-residency receipt is not. +- `gpuMemoryMiB`, on a price-list machine profile, is the memory of one of its + cards. + +A lane fits its residents when the footprints of its pinned identities, the +largest footprint of its lazy residents and the largest working memory any +resident needs at the lane's concurrency come to at most 80% of `gpuMemoryMiB`. +The planner puts an identity beside another only on a lane that fits them, of +one card per container and without fallback cells, on a profile that states +`gpuMemoryMiB`. A pinned identity beside other pinned identities is judged by a +co-residency receipt at the lane's cell, container shape and concurrency rather +than by a receipt of it alone: one whose `coResidents` include every other +identity pinned on the lane, since a receipt measured beside more residents +stands for fewer. Its warm bounds are checked and its demand is priced at that +receipt. The planner never places a generation identity, one that declares +`ttftP99Ms` and `itlP99Ms` or has a generate receipt, beside another identity, +nor another identity on a lane that hosts one. + ## Envelope A plan lane's envelope bounds what an online tuner may change without a new @@ -157,11 +191,7 @@ and its report. same bytes in whatever order the receipts, boot evidence, identities and demand series arrive. An identity that stays on its current placement keeps its current lane as it is, so a run whose `current_plan` already holds every -choice, such as its own previous plan, reports no change. The exceptions are -lazy residents that join a host lane one run later: a lane the previous run -created for an identity whose `comeAliveP95S` is `0` hosts lazy residents only -from the run after it, and an identity that left a host lane it was pinned on -may rejoin that lane as a lazy resident on the next run. +choice, such as its own previous plan, reports no change. | Parameter | Meaning | | --- | --- | @@ -216,7 +246,12 @@ document: since the host's container is already placed. The option costs what the identity's demand needs above the host lane's floor, plus the floor itself on a lane where no such identity stays pinned, since no other identity pays - it. A lane with fallback cells hosts no lazy resident. + it. The host must fit the identity beside the residents of its current + lane, as [Several models on one GPU](#several-models-on-one-gpu) states; + where a footprint or a working memory is not on file the option is a + measurement request for it. A lane with fallback cells, or one that hosts a + generation identity, hosts no lazy resident, and a generation identity is + never one. 5. A GPU product is unreliable when more than 5% of the cold attempts on it were `unplaced`, counting the environment's own attempts in place of borrowed ones. For an identity whose `comeAliveP95S` is at most @@ -234,21 +269,23 @@ The current placement is the identity's current lane when that lane has the option's cell and fallback cells, container shape and concurrency, the identity's warmth is the option's, and the lane's `queuePool` and `bundle` are the ones the parameters give the identity; a `lazy-resident` option must also be -hosted by that lane. A current placement that no option matches lacks evidence: -a receipt at its lane's cell, container shape and concurrency, a boot for a cold -warmth, or a load for a lazy resident. Unless it is known to miss a bound, -through a receipt at the lane that misses a warm bound, a measured come-alive -over the bound, a boot or load at the lane that did not succeed, a cold or -`lazy-resident` warmth under a bound of `0`, or an unreliable product without a -fallback cell on another product under a bound `hold_budget_s` fits, the -identity keeps that placement and leaves it only for an unblocked option at -least a cent cheaper than the placement. The placement is priced as an option of -its warmth at the lane's fastest receipt, or without one at its floor alone: one -container all month for `pinned`, for `lazy-resident` nothing beyond the lane's -floor, or the floor itself where no identity whose `comeAliveP95S` is `0` stays -pinned on the lane, and the burst tails for a cold warmth. What the placement -lacks becomes a measurement request. Such a placement on a lane the price list -cannot price has no cost to compare, so the identity keeps it. +hosted by that lane, and a `pinned` option on a lane where other identities are +pinned must stand on a co-residency receipt beside them. A current placement +that no option matches lacks evidence: a receipt at its lane's cell, container +shape and concurrency, a co-residency receipt beside the identities pinned with +it, a boot for a cold warmth, or a load for a lazy resident. Unless it is known +to miss a bound, through a receipt at the lane that misses a warm bound, a +measured come-alive over the bound, a boot or load at the lane that did not +succeed, a cold or `lazy-resident` warmth under a bound of `0`, or an unreliable +product without a fallback cell on another product under a bound `hold_budget_s` +fits, the identity keeps that placement and leaves it only for an unblocked +option at least a cent cheaper than the placement. The placement is priced as an +option of its warmth at the lane's fastest receipt, or without one at its floor +alone: one container all month for `pinned`, for `lazy-resident` nothing beyond +the lane's floor, or the floor itself where no identity whose `comeAliveP95S` is +`0` stays pinned on the lane, and the burst tails for a cold warmth. What the +placement lacks becomes a measurement request. Such a placement on a lane the +price list cannot price has no cost to compare, so the identity keeps it. An identity that stays on its current placement keeps that lane exactly as it is, beside whoever else the lane hosts: its cell, container shape, @@ -260,6 +297,36 @@ the identity added when that name is taken. The planner does not select current lane, capped by the reviewed ceiling, or stays out of the plan when it has none. +### Sharing a lane + +Once every identity has an option, the planner prices the plan as a whole and +changes it in the steps below. A floor is paid once per lane, so an option an +identity is offered alone can look cheaper than the lane it shares while the +plan total does not fall. + +1. A pinned identity whose co-pinned neighbours change in this run is decided + again beside the neighbours it is left with, until no identity's + neighbours change. +2. Identities that left a current lane return to it, one at a time and then + together, wherever their return does not raise the plan total. An identity + whose current placement is known to miss a bound does not return. +3. Packing. The planner visits every placed identity that is alone on its lane + and is not a generation identity, in decreasing order of its lane's floor + cost, ties broken by identity. For each it tries the other lanes of the plan + on its queue pool and bundle that keep a floor and host a pinned identity, + where it has a receipt at the lane's cell, container shape and concurrency: + first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, + since nothing moves that identity, then in decreasing order of the footprint + already on them, then by name. On the first lane where a move fits and lowers + the plan total by at least a cent, the identity joins it, as a lazy resident + when its bound admits its measured load onto the lane and selection did not + already offer it the lane, otherwise pinned, which needs every pinned + identity's co-residency receipt beside the others to meet its warm bounds. A + move that would lower the total but lacks a footprint, a working memory, a + load or a co-residency receipt is not made: it is a measurement request whose + `savingBound` is that fall. +4. Step 1 runs again for the lanes packing changed. + ### Pricing A container costs `(acceleratorPerHour * gpuCount + corePerHour * cpuCores + @@ -309,8 +376,10 @@ The report renders as a `placement-report` document: demand costs there with the lane's floor, the chosen cell, warmth, monthly cost and come-alive with its basis (`warm-floor`, `boot-evidence` or `borrowed-boot-evidence`), the `placementWaitS` part of a cold come-alive - where a placement wait was measured, the `fallbackCells` of a candidate list - and, for a lazy resident, its host `lane`, the peak containers its demand + where a placement wait was measured, the `fallbackCells` of a candidate list, + the `coResidents` of the co-residency receipt a co-pinned identity is judged + by and, for a resident that joins another identity's lane, its `lane`, the + peak containers its demand needs, and `nextCheaper`: the nearest option cheaper than the choice, or than a kept identity's current placement, that a service level blocks, the bounds that block it, and `saving`, the fall in the plan total found by planning @@ -342,7 +411,9 @@ The report renders as a `placement-report` document: and the monthly cost of its `floorMax`, which a floor budget prices. A lane on a cell the price list cannot price carries `unpricedReason` and no cost, and `unpricedIdentities` lists residents whose demand has no receipt to price - it on the lane. + it on the lane. A lane that hosts more than one identity names the GPU memory + its residents need, `memoryMiB`, and the 80% of its card's memory they may + use, `memoryLimitMiB`, where both are known. - `sharedChartLanes`: every chart lane, by `queuePool`, `machineProfile` and `bundle`, that more than one plan lane names. - `measurementRequests`: options that lack evidence, ranked by the saving they @@ -354,12 +425,18 @@ The report renders as a `placement-report` document: reason `failed`, or `unplaced` when no attempt got a container or the product's placement wait falls on an unplaced attempt, and a host lane without the identity's load asks for `lazy-resident` boot evidence on it, bounded by - what that option would cost. Savings are measured against the choice, or - against a kept identity's current placement. A request that cannot save is - dropped, and the requests of an identity without a placement or a current one - carry no bound and rank first. The evidence a kept current placement lacks is - requested at its lane, named in `lane` with its `concurrency`, and also ranks - first without a bound; a profile it asks for there is not asked for again. + what that option would cost, with `footprint` also asked for where a + resident's footprint or working memory is not on file. A packing that lacks + evidence asks at the lane it would join for `boot-evidence`, `footprint` or + `co-residency-receipt`, with the `coResidents` the receipt is to be measured + beside, bounded by the fall in the plan total the packing would bring. A kept + identity pinned beside others asks at its lane for a `co-residency-receipt` + beside them. Savings are measured against the choice, or against a kept + identity's current placement. A request that cannot save is dropped, and the + requests of an identity without a placement or a current one carry no bound + and rank first. The evidence a kept current placement lacks is requested at + its lane, named in `lane` with its `concurrency`, and also ranks first without + a bound; a profile it asks for there is not asked for again. - `comeAliveLevers`: cold and `lazy-resident` warmths whose measured come-alive misses the bound on a cell that meets the warm bounds for less than the choice, or than a kept identity's current placement, listed only when diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index bf286ef09..54dc9296b 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -65,6 +65,7 @@ BOOT_PERCENTILE_MIN_ROWS = 20 PLACEMENT_FAILURE_RATE_LIMIT = 0.05 PLACEMENT_FAILURE_FIELD = "placementFailureRate" +MEMORY_HEADROOM_FRACTION = 0.8 COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) _SLUG = re.compile(r"[^a-z0-9]+") @@ -262,6 +263,7 @@ class _Option: blockers: tuple[Blocker, ...] current: bool host: str | None = None + host_owner: str | None = None fallbacks: tuple[Receipt, ...] = () boot_s: float | None = None placement_wait_s: float | None = None @@ -305,9 +307,32 @@ def summary(self, saving: float | None = None) -> Option: lane=self.host, fallback_cells=self.cells[1:], placement_wait_s=self.placement_wait_s, + co_residents=self.receipt.co_residents, ) +@dataclass(frozen=True, slots=True, kw_only=True) +class _Memory: + fits: bool | None + need: int | None = None + limit: float | None = None + + +_UNPACKABLE = _Memory(fits=False) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Layout: + plan: Plan + names: dict[str, str] + kept: dict[str, tuple[PlanIdentity, PlanLane]] + receipts: dict[str, Receipt | None] + owners: dict[str, str] + + +type _CoKey = tuple[str, Cell, ContainerShape, int] + + @dataclass(frozen=True, slots=True, kw_only=True) class _Choice: identity: str @@ -321,6 +346,7 @@ class _Choice: gaps: tuple[MeasurementRequest, ...] = () unpriced: bool = False withdrawn: tuple[_Option, ...] = () + beside: frozenset[str] = frozenset() @property def baseline(self) -> float | None: @@ -544,14 +570,42 @@ def __init__( self.current = { entry.identity: (entry, self.current_lanes[entry.lane]) for entry in current_plan.identities } + self.current_pinned: defaultdict[str, set[str]] = defaultdict(set) + self.current_lazy: defaultdict[str, set[str]] = defaultdict(set) + if current_plan is not None: + for entry in current_plan.identities: + if entry.warmth is Warmth.PINNED: + self.current_pinned[entry.lane].add(entry.identity) + elif entry.warmth is Warmth.LAZY_RESIDENT: + self.current_lazy[entry.lane].add(entry.identity) self.refusals: list[Refusal] = [] self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} + self.co_receipts: defaultdict[_CoKey, list[tuple[str, Receipt]]] = defaultdict(list) self.receipts = self._accept_receipts(receipts) self.placements = _placement_index( record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None ) - self.boots = _boot_index(self._accept_boot_evidence(boot_evidence), self.placements) + accepted = self._accept_boot_evidence(boot_evidence) + self.boots = _boot_index(accepted, self.placements) + self.footprints: dict[tuple[str, Cell], int] = {} + self.working: dict[tuple[str, Cell, ContainerShape, int], int] = {} + measured = [receipt for pairs in (*self.receipts.values(), *self.co_receipts.values()) for _, receipt in pairs] + for receipt in measured: + self._footprint(receipt.identity, receipt.cell, receipt.footprint_mib) + if receipt.working_memory_mib is not None: + key = (receipt.identity, receipt.cell, receipt.container_shape, receipt.concurrency) + self.working[key] = max(receipt.working_memory_mib, self.working.get(key, 0)) + for record in accepted: + self._footprint(record.identity, record.cell, record.footprint_mib) + self.generation = {identity for identity, level in self.levels.items() if level.is_generation} + self.generation.update( + receipt.identity for receipt in measured if receipt.request_shape.operation is Operation.GENERATE + ) + + def _footprint(self, identity: str, cell: Cell, footprint: int | None) -> None: + if footprint is not None: + self.footprints[(identity, cell)] = max(footprint, self.footprints.get((identity, cell), 0)) def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[str, Receipt]]]: accepted: defaultdict[str, list[tuple[str, Receipt]]] = defaultdict(list) @@ -562,9 +616,15 @@ def _accept_receipts(self, receipts: Iterable[Receipt]) -> dict[str, list[tuple[ reason = _refusal_reason(receipt.cell, self.prices) or _incomparable_reason(level, receipt) if reason is not None: self._refuse(RefusedRecord.RECEIPT, receipt.identity, receipt.cell, receipt.source, reason) - self.refused_profiles.add((receipt.identity, receipt.cell.machine_profile)) + if not receipt.co_residents: + self.refused_profiles.add((receipt.identity, receipt.cell.machine_profile)) + continue + order = _canonical(receipt.to_dict()) + if receipt.co_residents: + key = (receipt.identity, receipt.cell, receipt.container_shape, receipt.concurrency) + self.co_receipts[key].append((order, receipt)) continue - accepted[receipt.identity].append((_canonical(receipt.to_dict()), receipt)) + accepted[receipt.identity].append((order, receipt)) return accepted def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvidence]: @@ -582,11 +642,21 @@ def _accept_boot_evidence(self, records: Iterable[BootEvidence]) -> list[BootEvi def _refuse(self, record: RefusedRecord, identity: str, cell: Cell, source: SourceRef, reason: str) -> None: self.refusals.append(Refusal(record=record, identity=identity, cell=cell, source=source, reason=reason)) - def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: tuple[Cell, ...] = ()) -> bool: + def is_current( + self, + identity: str, + receipt: Receipt, + warmth: Warmth, + fallbacks: tuple[Cell, ...] = (), + beside: frozenset[str] | None = None, + ) -> bool: current = self.current.get(identity) if current is None: return False entry, lane = current + context = self.pinned_beside(identity) if beside is None else beside + if warmth is Warmth.PINNED and not _stands_beside(receipt, context): + return False return ( entry.warmth is warmth and lane.queue_pool == self.parameters.queue_pool @@ -595,6 +665,58 @@ def is_current(self, identity: str, receipt: Receipt, warmth: Warmth, fallbacks: and _runs_on(receipt, lane) ) + def pinned_beside(self, identity: str) -> frozenset[str]: + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + return frozenset() + return frozenset(self.current_pinned[current[1].name] - {identity}) + + def co_receipt(self, identity: str, lane: PlanLane, beside: frozenset[str]) -> tuple[str, Receipt] | None: + found = [ + pair + for pair in self.co_receipts.get((identity, lane.cell, lane.container_shape, lane.concurrency), []) + if _stands_beside(pair[1], beside) + ] + return min( + found, + key=lambda pair: ( + len(pair[1].co_residents), + -_p99(pair[1]), + pair[1].throughput.requests_per_second, + pair[0], + ), + default=None, + ) + + def hosts_generation(self, identities: Iterable[str]) -> bool: + return any(identity in self.generation for identity in identities) + + def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> _Memory: + pinned, lazy = tuple(sorted(pinned)), tuple(sorted(lazy)) + price = self.prices.get(lane.cell.machine_profile) + residents = (*pinned, *lazy) + if ( + not residents + or price is None + or price.gpu_memory_mib is None + or lane.container_shape.gpu_count != 1 + or lane.fallback_cells + ): + return _UNPACKABLE + limit = price.gpu_memory_mib * MEMORY_HEADROOM_FRACTION + config = (lane.cell, lane.container_shape, lane.concurrency) + if any( + (identity, lane.cell) not in self.footprints or (identity, *config) not in self.working + for identity in residents + ): + return _Memory(fits=None, limit=limit) + need = ( + sum(self.footprints[(identity, lane.cell)] for identity in pinned) + + max((self.footprints[(identity, lane.cell)] for identity in lazy), default=0) + + max(self.working[(identity, *config)] for identity in residents) + ) + return _Memory(fits=need <= limit, need=need, limit=limit) + def hourly(self, receipt: Receipt) -> float: return container_hourly_cost( self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier @@ -667,6 +789,7 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: if (lazy if name == home else self.floor_paid(name, identity)) and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells + and not self.hosts_generation((*self.current_pinned[name], *self.current_lazy[name])) and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -707,12 +830,21 @@ def current_cost(self, identity: str, entry: PlanIdentity, lane: PlanLane) -> fl return hourly * hours def current_gaps( - self, identity: str, entry: PlanIdentity, lane: PlanLane, cost: float | None + self, + identity: str, + entry: PlanIdentity, + lane: PlanLane, + cost: float | None, + beside: frozenset[str] | None = None, ) -> tuple[MeasurementRequest, ...]: if cost is None: return () missing: list[tuple[Measurement, RequestReason]] = [] - if self.slowest_receipt(identity, lane) is None: + beside = self.pinned_beside(identity) if beside is None else beside + if beside: + if self.co_receipt(identity, lane, beside) is None: + missing.append((Measurement.CO_RESIDENCY_RECEIPT, RequestReason.UNMEASURED)) + elif self.slowest_receipt(identity, lane) is None: refused = (identity, lane.cell.machine_profile) in self.refused_profiles missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) if entry.warmth in BOOT_WARMTHS and (identity, lane.cell, lane.container_shape, entry.warmth) not in self.boots: @@ -729,11 +861,13 @@ def current_gaps( reason=reason, cost_bound=cost, saving_bound=None, + co_residents=tuple(sorted(beside)) if measurement is Measurement.CO_RESIDENCY_RECEIPT else (), ) for measurement, reason in missing ) - def choose(self, identity: str) -> _Choice: + def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice: + beside = self.pinned_beside(identity) if beside is None else beside level = self.levels[identity] requests = self.requests.get(identity, ()) options: list[_Option] = [] @@ -742,7 +876,7 @@ def choose(self, identity: str) -> _Choice: load_requests: dict[str, MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) - hosts = self.lazy_hosts(identity) + hosts = () if identity in self.generation else self.lazy_hosts(identity) for order, receipt in receipts: hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second @@ -759,7 +893,7 @@ def choose(self, identity: str) -> _Choice: come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, blockers=blocked, - current=self.is_current(identity, receipt, Warmth.PINNED), + current=self.is_current(identity, receipt, Warmth.PINNED, beside=beside), ) ) cold_cost = hourly * self.window.cold_container_hours(requests, rate) @@ -815,7 +949,18 @@ def choose(self, identity: str) -> _Choice: continue lazy_cost = hourly * self.resident_hours(host, identity, requests, rate) load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) - if load is None or load.seconds is None: + memory = self.memory( + host, + pinned=self.current_pinned[host.name] - {identity}, + lazy=self.current_lazy[host.name] | {identity}, + ) + if memory.fits is False: + continue + missing = ( + *(() if load is not None and load.seconds is not None else (Measurement.BOOT_EVIDENCE,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + if missing: known = load_requests.get(host.name) if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): load_requests[host.name] = MeasurementRequest( @@ -824,13 +969,19 @@ def choose(self, identity: str) -> _Choice: container_shape=host.container_shape, concurrency=host.concurrency, lane=host.name, - measure=(Measurement.BOOT_EVIDENCE,), + measure=missing, warmth=Warmth.LAZY_RESIDENT, - reason=RequestReason.UNMEASURED if load is None else RequestReason.FAILED, + reason=( + RequestReason.FAILED + if load is not None and load.seconds is None + else RequestReason.UNMEASURED + ), cost_bound=lazy_cost, saving_bound=None, ) continue + if load is None or load.seconds is None: + continue blocker = _come_alive_blocker(level, load.seconds) options.append( _Option( @@ -847,8 +998,9 @@ def choose(self, identity: str) -> _Choice: boot_s=load.boot_s, ) ) + options.extend(self.beside_options(identity, level, requests, current_class, beside)) if self.tight(level): - options.extend(self.candidate_lists(identity, level, receipts, requests, current_class)) + options.extend(self.candidate_lists(identity, level, receipts, requests, current_class, beside)) feasible = [option for option in options if option.feasible] chosen = min(feasible, key=lambda option: option.rank) if feasible else None held = None @@ -864,7 +1016,7 @@ def choose(self, identity: str) -> _Choice: if chosen is None: held = cost unpriced = cost is None - gaps = self.current_gaps(identity, entry, lane, cost) + gaps = self.current_gaps(identity, entry, lane, cost, beside) gapped = { (request.cell, request.container_shape, request.warmth) for request in gaps @@ -886,8 +1038,44 @@ def choose(self, identity: str) -> _Choice: gaps=gaps, unpriced=unpriced, withdrawn=tuple(withdrawn), + beside=beside, ) + def beside_options( + self, + identity: str, + level: ServiceLevel, + requests: Sequence[int], + current_class: Cell | None, + beside: frozenset[str], + ) -> list[_Option]: + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + return [] + lane = current[1] + if not beside and any(_runs_on(receipt, lane) for _, receipt in self.receipts.get(identity, [])): + return [] + found = self.co_receipt(identity, lane, beside) + if found is None or not self.is_current(identity, found[1], Warmth.PINNED, beside=beside): + return [] + order, receipt = found + return [ + _Option( + receipt=receipt, + order=order, + warmth=Warmth.PINNED, + cost=self.hourly(receipt) + * self.window.warm_container_hours(requests, receipt.throughput.requests_per_second), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=( + *self.warm_blockers(level, receipt, current_class), + *self.reliability_blockers(level, lane.cells), + ), + current=True, + ) + ] + def candidate_lists( self, identity: str, @@ -895,6 +1083,7 @@ def candidate_lists( receipts: Sequence[tuple[str, Receipt]], requests: Sequence[int], current_class: Cell | None, + beside: frozenset[str], ) -> list[_Option]: lists = [] for order, receipt in receipts: @@ -921,7 +1110,7 @@ def candidate_lists( come_alive_s=0.0, basis=ComeAliveBasis.WARM_FLOOR, blockers=(), - current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,)), + current=self.is_current(identity, receipt, Warmth.PINNED, (second.cell,), beside), ) ) return lists @@ -929,7 +1118,9 @@ def candidate_lists( def receipt_requests(self, choice: _Choice, baseline: float | None) -> list[MeasurementRequest]: measured = {receipt.cell.machine_profile for _, receipt in self.receipts.get(choice.identity, [])} measured.update( - request.cell.machine_profile for request in choice.gaps if Measurement.RECEIPT in request.measure + request.cell.machine_profile + for request in choice.gaps + if Measurement.RECEIPT in request.measure or Measurement.CO_RESIDENCY_RECEIPT in request.measure ) bound = choice.level.come_alive_p95_s tight = self.tight(choice.level) @@ -1254,15 +1445,12 @@ def demand_hours(self, lane: PlanLane, identity: str, receipt: Receipt | None) - return window.above_floor_hours(requests, rate, lane.envelope.floor_min) return window.cold_container_hours(requests, rate) - def run(self, *, price_savings: bool = True) -> PlannerResult: - never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] - self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) - choices = [self.choose(identity) for identity in sorted(self.levels)] - placed = {choice.identity: choice.chosen for choice in choices if choice.chosen is not None} + def assemble(self, choices: Mapping[str, _Choice]) -> _Layout: + placed = {identity: choice.chosen for identity, choice in choices.items() if choice.chosen is not None} kept = { - choice.identity: self.current[choice.identity] - for choice in choices - if choice.chosen is None and choice.identity in self.current + identity: self.current[identity] + for identity, choice in choices.items() + if choice.chosen is None and identity in self.current } staying = {identity: self.current[identity][1].name for identity, option in placed.items() if option.current} joining = { @@ -1270,13 +1458,19 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: for identity, option in placed.items() if (host := option.host) is not None and identity not in staying } + riding = { + identity: owner + for identity, option in placed.items() + if (owner := option.host_owner) is not None and identity not in staying + } reserved = {lane.name for _, lane in kept.values()} | set(staying.values()) | set(joining.values()) moving = { identity: option for identity, option in placed.items() - if identity not in staying and identity not in joining + if identity not in staying and identity not in joining and identity not in riding } names = staying | joining | self.lane_names(moving, reserved) + names.update((identity, names[owner]) for identity, owner in riding.items()) carried = [lane for _, lane in kept.values()] carried.extend(self.current_lanes[name] for name in (*staying.values(), *joining.values())) lanes = {lane.name: self.within_ceiling(lane) for lane in carried} @@ -1287,17 +1481,334 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: PlanIdentity(identity=identity, lane=names[identity], warmth=option.warmth) for identity, option in placed.items() ) - new_plan = Plan( + plan = Plan( placement=self.parameters.placement, lanes=tuple(lanes[name] for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} receipts.update({identity: self.slowest_receipt(identity, lane) for identity, (_, lane) in kept.items()}) + return _Layout( + plan=plan, + names=names, + kept=kept, + receipts=receipts, + owners={names[identity]: identity for identity in moving}, + ) + + def total(self, layout: _Layout) -> float: + lane_reports, _ = self._pricing(layout.plan, layout.receipts) + return sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) + + def stay(self, choice: _Choice) -> _Choice | None: + identity = choice.identity + current = self.keepable(identity) + if current is None: + return None + standing = [option for option in choice.options if option.current] + feasible = [option for option in standing if option.feasible] + if feasible: + return replace(choice, chosen=min(feasible, key=lambda option: option.rank)) + entry, lane = current + if standing or self.kept_blockers(identity, entry, lane, self.slowest_receipt(identity, lane)): + return None + cost = self.current_cost(identity, entry, lane) + gaps = self.current_gaps(identity, entry, lane, cost, choice.beside) + gapped = { + (request.cell, request.container_shape, request.warmth) + for request in gaps + if Measurement.BOOT_EVIDENCE in request.measure + } + return replace( + choice, + chosen=None, + held=cost, + unpriced=cost is None, + gaps=gaps, + boot_requests=tuple( + request + for request in choice.boot_requests + if (request.cell, request.container_shape, request.warmth) not in gapped + ), + ) + + @staticmethod + def stays(identity: str, choice: _Choice, layout: _Layout) -> bool: + if choice.chosen is None: + return identity in layout.kept + return choice.chosen.current + + def stay_beside(self, choices: dict[str, _Choice], originals: Mapping[str, _Choice]) -> dict[str, _Choice]: + layout = self.assemble(choices) + total = self.total(layout) + left: defaultdict[str, dict[str, _Choice]] = defaultdict(dict) + for identity in sorted(choices): + if identity not in self.current or self.stays(identity, choices[identity], layout): + continue + stay = self.stay(originals[identity]) + if stay is not None: + left[self.current[identity][1].name][identity] = stay + for lane in sorted(left): + movers = left[lane] + groups = [{identity: stay} for identity, stay in movers.items()] + if len(movers) > 1: + groups.append(movers) + for group in groups: + trial = self.settle({**choices, **group}) + trial_layout = self.assemble(trial) + trial_total = self.total(trial_layout) + if round(trial_total - total, 2) <= 0: + choices, layout, total = trial, trial_layout, trial_total + return choices + + def riding_cost(self, lane: PlanLane, identity: str, receipt: Receipt) -> float: + window = replace(self.window, ceiling=lane.max_containers) + requests = self.requests.get(identity, ()) + rate = receipt.throughput.requests_per_second + return self.lane_hourly(lane) * window.above_floor_hours(requests, rate, lane.envelope.floor_min) + + def in_use(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> int: + pinned_use = sum(self.footprints.get((identity, lane.cell), 0) for identity in pinned) + return pinned_use + max((self.footprints.get((identity, lane.cell), 0) for identity in lazy), default=0) + + def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, frozenset[str], frozenset[str]]]: + pinned: defaultdict[str, set[str]] = defaultdict(set) + lazy: defaultdict[str, set[str]] = defaultdict(set) + for entry in layout.plan.identities: + (pinned if entry.warmth is Warmth.PINNED else lazy)[entry.lane].add(entry.identity) + targets = [ + (lane, frozenset(pinned[lane.name]), frozenset(lazy[lane.name])) + for lane in layout.plan.lanes + if lane.name != layout.names[identity] + and lane.envelope.floor_min >= PINNED_FLOOR + and not lane.fallback_cells + and lane.container_shape.gpu_count == 1 + and lane.queue_pool == self.parameters.queue_pool + and lane.bundle == self.parameters.bundle_of(identity) + and pinned[lane.name] + and not self.hosts_generation((*pinned[lane.name], *lazy[lane.name])) + and self.slowest_receipt(identity, lane) is not None + ] + return sorted( + targets, + key=lambda target: ( + not any(self.levels[resident].come_alive_p95_s == 0 for resident in target[1]), + -self.in_use(*target), + target[0].name, + ), + ) + + def packings( + self, + identity: str, + target: tuple[PlanLane, frozenset[str], frozenset[str]], + layout: _Layout, + choices: Mapping[str, _Choice], + ) -> list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]]: + lane, pinned, lazy = target + level = self.levels[identity] + order, receipt = min( + (pair for pair in self.receipts.get(identity, []) if _runs_on(pair[1], lane)), + key=lambda pair: (pair[1].throughput.requests_per_second, pair[0]), + ) + owner = layout.owners.get(lane.name) + host = None if owner is not None else lane.name + offered = choices[identity] + hosts = {option.host for option in offered.options if option.warmth is Warmth.LAZY_RESIDENT} + hosts.update(request.lane for request in offered.boot_requests if request.warmth is Warmth.LAZY_RESIDENT) + packings: list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]] = [] + current_class = self.current_class(identity) + blocked = (*self.warm_blockers(level, receipt, current_class), *self.reliability_blockers(level, lane.cells)) + load = self.boots.get((identity, lane.cell, lane.container_shape, Warmth.LAZY_RESIDENT)) + if ( + level.come_alive_p95_s != 0 + and not blocked + and lane.name not in hosts + and (load is None or load.seconds is not None) + and (load is None or load.seconds is None or _come_alive_blocker(level, load.seconds) is None) + ): + memory = self.memory(lane, pinned=pinned, lazy=lazy | {identity}) + if memory.fits is not False: + missing = ( + *(() if load is not None else (Measurement.BOOT_EVIDENCE,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + option = _Option( + receipt=receipt, + order=order, + warmth=Warmth.LAZY_RESIDENT, + cost=self.riding_cost(lane, identity, receipt), + come_alive_s=0.0 if load is None or load.seconds is None else load.seconds, + basis=ComeAliveBasis.BOOT_EVIDENCE if load is None else load.basis, + blockers=(), + current=False, + boot_s=None if load is None else load.boot_s, + host=host, + host_owner=owner, + ) + packings.append(({identity: option}, missing, ())) + members = pinned | {identity} + if all(choices[member].chosen is not None for member in pinned): + together = {member: self.co_receipt(member, lane, members - {member}) for member in members} + measured = {member: pair for member, pair in together.items() if pair is not None} + if not any( + ( + *self.warm_blockers(self.levels[member], pair[1], self.current_class(member)), + *self.reliability_blockers(self.levels[member], lane.cells), + ) + for member, pair in measured.items() + ): + memory = self.memory(lane, pinned=members, lazy=lazy) + if memory.fits is not False: + missing = ( + *(() if len(measured) == len(members) else (Measurement.CO_RESIDENCY_RECEIPT,)), + *(() if memory.fits else (Measurement.FOOTPRINT,)), + ) + own_order, own = measured.get(identity, (order, receipt)) + updates = { + identity: _Option( + receipt=own, + order=own_order, + warmth=Warmth.PINNED, + cost=self.riding_cost(lane, identity, own), + come_alive_s=0.0, + basis=ComeAliveBasis.WARM_FLOOR, + blockers=(), + current=False, + host=host, + host_owner=owner, + ) + } + if not missing: + for member in sorted(pinned): + chosen = choices[member].chosen + member_order, member_receipt = measured[member] + if chosen is not None: + rate = member_receipt.throughput.requests_per_second + updates[member] = replace( + chosen, + receipt=member_receipt, + order=member_order, + cost=self.hourly(member_receipt) + * self.window.warm_container_hours(choices[member].requests, rate), + ) + packings.append((updates, missing, tuple(sorted(pinned)))) + return packings + + def pack(self, choices: dict[str, _Choice]) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: + layout = self.assemble(choices) + total = self.total(layout) + lanes = {lane.name: lane for lane in layout.plan.lanes} + hosted = Counter(entry.lane for entry in layout.plan.identities) + hours = self.parameters.hours_per_month + + def floor_cost(identity: str) -> float: + lane = lanes[layout.names[identity]] + if _lane_refusal(lane, self.prices) is not None: + return 0.0 + return round(self.lane_hourly(lane) * lane.envelope.floor_min * hours, 6) + + candidates = [ + identity + for identity, choice in choices.items() + if (chosen := choice.chosen) is not None + and chosen.host is None + and chosen.host_owner is None + and chosen.warmth is not Warmth.LAZY_RESIDENT + and identity not in self.generation + and hosted[layout.names[identity]] == 1 + ] + requests: dict[tuple[str, str, Warmth], MeasurementRequest] = {} + for identity in sorted(candidates, key=lambda identity: (-floor_cost(identity), identity)): + if hosted.get(layout.names.get(identity, ""), 0) != 1: + continue + moved = False + for target in self.pack_targets(identity, layout): + for updates, missing, beside in self.packings(identity, target, layout, choices): + members = frozenset(beside) | {identity} + trial = { + **choices, + **{ + member: replace(choices[member], chosen=option) + if member == identity + else replace(choices[member], chosen=option, beside=members - {member}) + for member, option in updates.items() + }, + } + trial_layout = self.assemble(trial) + trial_total = self.total(trial_layout) + saving = total - trial_total + if round(saving, 2) <= 0: + continue + option = updates[identity] + if missing: + key = (identity, target[0].name, option.warmth) + known = requests.get(key) + if known is None or saving > (known.saving_bound or 0.0): + requests[key] = MeasurementRequest( + identity=identity, + cell=target[0].cell, + container_shape=target[0].container_shape, + concurrency=target[0].concurrency, + lane=target[0].name, + measure=missing, + warmth=option.warmth, + reason=RequestReason.UNMEASURED, + cost_bound=option.cost, + saving_bound=saving, + co_residents=beside, + ) + continue + choices, layout, total = trial, trial_layout, trial_total + hosted = Counter(entry.lane for entry in layout.plan.identities) + moved = True + break + if moved: + break + return choices, list(requests.values()) + + def stale_beside(self, choices: Mapping[str, _Choice]) -> dict[str, frozenset[str]]: + layout = self.assemble(choices) + pinned: defaultdict[str, set[str]] = defaultdict(set) + for entry in layout.plan.identities: + if entry.warmth is Warmth.PINNED: + pinned[entry.lane].add(entry.identity) + stale: dict[str, frozenset[str]] = {} + for identity, choice in choices.items(): + current = self.current.get(identity) + if current is None or current[0].warmth is not Warmth.PINNED: + continue + if self.stays(identity, choice, layout): + beside = frozenset(pinned[current[1].name] - {identity}) + if beside != choice.beside: + stale[identity] = beside + return stale + + def settle(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: + for _ in range(len(self.levels) + 1): + stale = self.stale_beside(choices) + if not stale: + return choices + choices = {**choices, **{identity: self.choose(identity, beside) for identity, beside in stale.items()}} + return choices + + def decide(self) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: + never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] + self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) + originals = {identity: self.choose(identity) for identity in sorted(self.levels)} + decided = self.stay_beside(self.settle(dict(originals)), originals) + decided, requests = self.pack(decided) + return self.settle(decided), requests + + def run(self, *, price_savings: bool = True) -> PlannerResult: + decided, packing_requests = self.decide() + choices = [decided[identity] for identity in sorted(decided)] + layout = self.assemble(decided) + new_plan, names, kept, receipts = layout.plan, layout.names, layout.kept, layout.receipts lane_reports, costs = self._pricing(new_plan, receipts) total = sum(lane.monthly_cost for lane in lane_reports if lane.monthly_cost is not None) reports: list[IdentityReport] = [] - requests: list[MeasurementRequest] = [] + requests: list[MeasurementRequest] = list(packing_requests) levers: list[Lever] = [] scarcity: list[Scarcity] = [] for choice in choices: @@ -1374,12 +1885,15 @@ def _identity_report( ) return replace(report, unplaced_reason=reason) rate = chosen.pricing_receipt.throughput.requests_per_second + summary = chosen.summary() + if chosen.host_owner is not None: + summary = replace(summary, lane=names[choice.identity]) return replace( report, status=IdentityStatus.PLACED, lane=names[choice.identity], monthly_cost=cost, - chosen=chosen.summary(), + chosen=summary, peak_containers=max((_Window.needed(count, rate) for count in choice.requests), default=0), next_cheaper=next_cheaper, ) @@ -1388,6 +1902,7 @@ def _pricing( self, plan: Plan, receipts: Mapping[str, Receipt | None] ) -> tuple[tuple[LaneReport, ...], dict[str, float]]: residents: defaultdict[str, list[str]] = defaultdict(list) + warmths = {entry.identity: entry.warmth for entry in plan.identities} for entry in plan.identities: residents[entry.lane].append(entry.identity) hours = self.parameters.hours_per_month @@ -1404,6 +1919,13 @@ def _pricing( demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} costs.update((identity, hourly * (floor + value)) for identity, value in priced.items()) + memory = _UNPACKABLE + if len(hosted) > 1: + memory = self.memory( + lane, + pinned=(identity for identity in hosted if warmths[identity] is Warmth.PINNED), + lazy=(identity for identity in hosted if warmths[identity] is Warmth.LAZY_RESIDENT), + ) reports.append( LaneReport( name=lane.name, @@ -1413,6 +1935,8 @@ def _pricing( floor_monthly_cost=hourly * lane.envelope.floor_min * hours, envelope_max_monthly_cost=hourly * lane.envelope.floor_max * hours, unpriced_identities=tuple(identity for identity in hosted if identity not in priced), + memory_mib=memory.need, + memory_limit_mib=memory.limit if memory.need is not None else None, ) ) return tuple(reports), costs @@ -1429,6 +1953,16 @@ def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | Non return ScarcityReason.PLACEMENT_WAIT +def _stands_beside(receipt: Receipt, beside: frozenset[str]) -> bool: + return beside <= frozenset(receipt.co_residents) if receipt.co_residents else not beside + + +def _p99(receipt: Receipt) -> float: + return max( + percentiles.p99_ms for percentiles in (receipt.latency, receipt.ttft, receipt.itl) if percentiles is not None + ) + + def _saving(total: float, replanned: float) -> float: saving = total - replanned return saving if round(saving, 2) > 0 else 0.0 diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index d63ffc25d..0f0935ce2 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -45,6 +45,8 @@ class ComeAliveBasis(StrEnum): class Measurement(StrEnum): RECEIPT = "receipt" BOOT_EVIDENCE = "boot-evidence" + FOOTPRINT = "footprint" + CO_RESIDENCY_RECEIPT = "co-residency-receipt" class RequestReason(StrEnum): @@ -108,6 +110,7 @@ class Option: lane: str | None = None fallback_cells: tuple[Cell, ...] = () placement_wait_s: float | None = None + co_residents: tuple[str, ...] = () def to_dict(self) -> dict[str, object]: return _compact( @@ -117,6 +120,7 @@ def to_dict(self) -> dict[str, object]: "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, + "coResidents": list(self.co_residents) or None, "warmth": self.warmth.value, "monthlyCost": _money(self.monthly_cost), "saving": None if self.saving is None else _money(self.saving), @@ -173,6 +177,8 @@ class LaneReport: envelope_max_monthly_cost: float | None = None unpriced_reason: str | None = None unpriced_identities: tuple[str, ...] = () + memory_mib: int | None = None + memory_limit_mib: float | None = None @property def unpriced(self) -> bool: @@ -191,6 +197,8 @@ def to_dict(self) -> dict[str, object]: ), "unpricedReason": self.unpriced_reason, "unpricedIdentities": list(self.unpriced_identities) or None, + "memoryMiB": self.memory_mib, + "memoryLimitMiB": self.memory_limit_mib, } ) @@ -223,6 +231,7 @@ class MeasurementRequest: container_shape: ContainerShape | None = None concurrency: int | None = None lane: str | None = None + co_residents: tuple[str, ...] = () def to_dict(self) -> dict[str, object]: return _compact( @@ -232,6 +241,7 @@ def to_dict(self) -> dict[str, object]: "containerShape": None if self.container_shape is None else self.container_shape.to_dict(), "concurrency": self.concurrency, "lane": self.lane, + "coResidents": list(self.co_residents) or None, "measure": [measurement.value for measurement in self.measure], "warmth": self.warmth.value, "reason": self.reason.value, diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index 93b9d9cd0..9248f33d6 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -137,6 +137,12 @@ def _string(self, key: str, value: object) -> str: raise SchemaError(self._at(key), f"expected a string, got {_describe(value)}") return value + @staticmethod + def text(value: object, path: str) -> str: + if not isinstance(value, str): + raise SchemaError(path, f"expected a string, got {_describe(value)}") + return value + def _boolean(self, key: str, value: object) -> bool: if not isinstance(value, bool): raise SchemaError(self._at(key), f"expected a boolean, got {_describe(value)}") @@ -605,10 +611,21 @@ class Receipt: latency: Percentiles | None = None ttft: Percentiles | None = None itl: Percentiles | None = None + footprint_mib: int | None = None + working_memory_mib: int | None = None + co_residents: tuple[str, ...] = () def __post_init__(self) -> None: _require_identifier("identity", self.identity) _require_at_least("concurrency", self.concurrency, 1) + _require_at_least("footprintMiB", self.footprint_mib, 1) + _require_at_least("workingMemoryMiB", self.working_memory_mib, 0) + for index, resident in enumerate(self.co_residents): + _require_identifier(f"coResidents[{index}]", resident) + if resident == self.identity: + raise SchemaError(f"coResidents[{index}]", f"names the receipt's own identity {resident!r}") + if resident in self.co_residents[:index]: + raise SchemaError(f"coResidents[{index}]", f"duplicates {resident!r}") if self.request_shape.operation is Operation.GENERATE: _require_absent("latency", self.latency, "for the generate operation, which reports ttft and itl") if self.ttft is None: @@ -641,6 +658,9 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: latency=reader.optional_child("latency", Percentiles.from_dict), ttft=reader.optional_child("ttft", Percentiles.from_dict), itl=reader.optional_child("itl", Percentiles.from_dict), + footprint_mib=reader.optional_integer("footprintMiB"), + working_memory_mib=reader.optional_integer("workingMemoryMiB"), + co_residents=reader.children("coResidents", _Reader.text, required=False), source=reader.child("source", SourceRef.from_dict), ) @@ -653,11 +673,14 @@ def to_dict(self) -> dict[str, object]: "cell": self.cell.to_dict(), "containerShape": self.container_shape.to_dict(), "concurrency": self.concurrency, + "coResidents": list(self.co_residents) or None, "requestShape": self.request_shape.to_dict(), "throughput": self.throughput.to_dict(), "latency": None if self.latency is None else self.latency.to_dict(), "ttft": None if self.ttft is None else self.ttft.to_dict(), "itl": None if self.itl is None else self.itl.to_dict(), + "footprintMiB": self.footprint_mib, + "workingMemoryMiB": self.working_memory_mib, "source": self.source.to_dict(), } ) @@ -673,6 +696,7 @@ class BootEvidence: source: SourceRef boot_seconds: float | None = None placement_wait_s: float | None = None + footprint_mib: int | None = None borrowed: bool = False def __post_init__(self) -> None: @@ -692,6 +716,9 @@ def __post_init__(self) -> None: raise SchemaError("bootSeconds", "is required for an attempt that was placed") _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) + _require_at_least("footprintMiB", self.footprint_mib, 1) + if self.outcome is not BootOutcome.SUCCEEDED: + _require_absent("footprintMiB", self.footprint_mib, "for an attempt that did not succeed") @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -706,6 +733,7 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: placement_wait_s=reader.optional_number("placementWaitS"), boot_seconds=reader.optional_number("bootSeconds"), outcome=reader.enum("outcome", BootOutcome), + footprint_mib=reader.optional_integer("footprintMiB"), borrowed=reader.optional_boolean("borrowed") is True, source=reader.child("source", SourceRef.from_dict), ) @@ -722,6 +750,7 @@ def to_dict(self) -> dict[str, object]: "placementWaitS": self.placement_wait_s, "bootSeconds": self.boot_seconds, "outcome": self.outcome.value, + "footprintMiB": self.footprint_mib, "borrowed": True if self.borrowed else None, "source": self.source.to_dict(), } @@ -735,6 +764,7 @@ class MachineProfilePrice: accelerator_per_hour: float core_per_hour: float memory_gib_per_hour: float + gpu_memory_mib: int | None = None def __post_init__(self) -> None: _require_identifier("name", self.name) @@ -742,6 +772,7 @@ def __post_init__(self) -> None: _require_at_least("acceleratorPerHour", self.accelerator_per_hour, 0) _require_at_least("corePerHour", self.core_per_hour, 0) _require_at_least("memoryGiBPerHour", self.memory_gib_per_hour, 0) + _require_at_least("gpuMemoryMiB", self.gpu_memory_mib, 1) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: @@ -750,19 +781,23 @@ def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: cls, name=reader.string("name"), gpu_product=reader.string("gpuProduct"), + gpu_memory_mib=reader.optional_integer("gpuMemoryMiB"), accelerator_per_hour=reader.number("acceleratorPerHour"), core_per_hour=reader.number("corePerHour"), memory_gib_per_hour=reader.number("memoryGiBPerHour"), ) def to_dict(self) -> dict[str, object]: - return { - "name": self.name, - "gpuProduct": self.gpu_product, - "acceleratorPerHour": self.accelerator_per_hour, - "corePerHour": self.core_per_hour, - "memoryGiBPerHour": self.memory_gib_per_hour, - } + return _compact( + { + "name": self.name, + "gpuProduct": self.gpu_product, + "gpuMemoryMiB": self.gpu_memory_mib, + "acceleratorPerHour": self.accelerator_per_hour, + "corePerHour": self.core_per_hour, + "memoryGiBPerHour": self.memory_gib_per_hour, + } + ) @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index de243df5b..91a4f0861 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -47,13 +47,15 @@ "memoryMiB": 16384 }, "concurrency": 16, + "coResidents": [ + "intfloat/e5-base-v2" + ], "warmth": "pinned", "monthlyCost": 759.2, "comeAliveS": 0.0, "comeAliveBasis": "warm-floor", "receipt": { - "uri": "receipts/baai-bge-m3/l4-c16-16384.json", - "sha256": "0000000000000000000000000000000000000000000000000000000000000001" + "uri": "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" } }, "peakContainers": 1, @@ -407,8 +409,11 @@ }, "concurrency": 16, "lane": "default-l4-default", + "coResidents": [ + "BAAI/bge-m3" + ], "measure": [ - "receipt" + "co-residency-receipt" ], "warmth": "pinned", "reason": "unmeasured", @@ -477,7 +482,8 @@ "concurrency": 16, "lane": "default-l4-default", "measure": [ - "boot-evidence" + "boot-evidence", + "footprint" ], "warmth": "lazy-resident", "reason": "unmeasured", diff --git a/packages/sie_placement/tests/fixtures/planner/price-list.json b/packages/sie_placement/tests/fixtures/planner/price-list.json index 08d835d8a..871410550 100644 --- a/packages/sie_placement/tests/fixtures/planner/price-list.json +++ b/packages/sie_placement/tests/fixtures/planner/price-list.json @@ -6,6 +6,7 @@ { "name": "l4", "gpuProduct": "NVIDIA-L4", + "gpuMemoryMiB": 23034, "acceleratorPerHour": 0.8, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -13,6 +14,7 @@ { "name": "l40s", "gpuProduct": "NVIDIA-L40S", + "gpuMemoryMiB": 46068, "acceleratorPerHour": 1.2, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -20,6 +22,7 @@ { "name": "a100-80gb", "gpuProduct": "NVIDIA-A100-SXM4-80GB", + "gpuMemoryMiB": 81920, "acceleratorPerHour": 2.5, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 @@ -27,6 +30,7 @@ { "name": "h100", "gpuProduct": "NVIDIA-H100-80GB-HBM3", + "gpuMemoryMiB": 81559, "acceleratorPerHour": 4.0, "corePerHour": 0.04, "memoryGiBPerHour": 0.005 diff --git a/packages/sie_placement/tests/fixtures/planner/receipts.json b/packages/sie_placement/tests/fixtures/planner/receipts.json index c29fe2621..31c080c66 100644 --- a/packages/sie_placement/tests/fixtures/planner/receipts.json +++ b/packages/sie_placement/tests/fixtures/planner/receipts.json @@ -26,11 +26,49 @@ "p95Ms": 171.9, "p99Ms": 214 }, + "footprintMiB": 2310, + "workingMemoryMiB": 1480, "source": { "uri": "receipts/baai-bge-m3/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000001" } }, + { + "kind": "placement-receipt", + "schemaVersion": 1, + "identity": "BAAI/bge-m3", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "coResidents": [ + "intfloat/e5-base-v2" + ], + "requestShape": { + "operation": "encode", + "itemsPerRequest": 8, + "inputTokensPerItem": 256 + }, + "throughput": { + "requestsPerSecond": 36 + }, + "latency": { + "p50Ms": 110.2, + "p95Ms": 171.9, + "p99Ms": 236 + }, + "footprintMiB": 2310, + "workingMemoryMiB": 1720, + "source": { + "uri": "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" + } + }, { "kind": "placement-receipt", "schemaVersion": 1, @@ -89,6 +127,8 @@ "p50Ms": 101, "p99Ms": 190 }, + "footprintMiB": 2290, + "workingMemoryMiB": 1210, "source": { "uri": "receipts/baai-bge-m3-sparse/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000003" @@ -194,6 +234,8 @@ "latency": { "p99Ms": 180 }, + "footprintMiB": 1650, + "workingMemoryMiB": 980, "source": { "uri": "receipts/qwen-qwen3-reranker-0.6b/l4-c16-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000006" @@ -225,6 +267,8 @@ "p50Ms": 140, "p99Ms": 280 }, + "footprintMiB": 8420, + "workingMemoryMiB": 2650, "source": { "uri": "receipts/qwen-qwen3-embedding-4b/l4-c8-16384.json", "sha256": "0000000000000000000000000000000000000000000000000000000000000007" diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index eac0a3852..7dca28e73 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -55,6 +55,7 @@ A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") H100 = Cell(machine_profile="h100", gpu_product="NVIDIA-H100-80GB-HBM3") RATES = {L4: 1.0, A100: 3.0, H100: 4.0} +CARD_MIB = {L4: 23034, A100: 81920, H100: 81559} SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) IDENTITY = "org/model" @@ -68,6 +69,7 @@ accelerator_per_hour=rate, core_per_hour=0.0, memory_gib_per_hour=0.0, + gpu_memory_mib=CARD_MIB[cell], ) for cell, rate in RATES.items() ), @@ -76,7 +78,9 @@ UNRELIABLE = 1 / 3 -def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0) -> Receipt: +def receipt( + identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0, footprint=None, working=None +) -> Receipt: return Receipt( identity=identity, cell=cell, @@ -85,6 +89,8 @@ def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), + footprint_mib=footprint, + working_memory_mib=working, source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{rps}/{p99}"), ) @@ -302,7 +308,8 @@ def test_a_lazy_resident_load_waits_for_no_placement(): ) service_levels = levels(**{OTHER: 0, IDENTITY: 60}) boots = [attempt(warmth=Warmth.LAZY_RESIDENT, boot=20.0), attempt(OTHER, boot=1.0, wait=500.0, index=1)] - result = run(service_levels, [receipt(OTHER), receipt()], boots, current=current) + receipts = [receipt(OTHER, footprint=4000, working=1000), receipt(footprint=2000, working=1000)] + result = run(service_levels, receipts, boots, current=current) chosen = result.report.identity(IDENTITY).chosen assert (chosen.warmth, chosen.lane, chosen.come_alive_s, chosen.placement_wait_s) == ( Warmth.LAZY_RESIDENT, diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index feed8adf7..aec54c2bf 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -59,7 +59,7 @@ L4_HOURLY = 1.0 + 4 * 0.1 -def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10.0): +def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10.0, beside=()): return Receipt( identity=identity, cell=cell, @@ -68,7 +68,10 @@ def receipt(identity, cell=L4, *, shape=SHAPE, p99=100.0, concurrency=8, rps=10. request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), - source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}"), + co_residents=tuple(beside), + source=SourceRef( + uri=f"receipts/{identity}/{cell.machine_profile}/{shape.memory_mib}/{concurrency}/{'+'.join(beside)}" + ), ) @@ -265,15 +268,22 @@ def test_an_identity_staying_beside_a_neighbour_offers_no_lever_that_loosening_i bound, seconds, permitted ): bounds = {"org/a": bound, "org/b": 0} - receipts = [receipt("org/a"), receipt("org/b")] + receipts = [ + receipt("org/a"), + receipt("org/b"), + receipt("org/a", beside=["org/b"]), + receipt("org/b", beside=["org/a"]), + ] boots = [boot("org/a", seconds)] result = planned(bounds, receipts, boots, current=shared_lane("org/a", "org/b")) assert result.plan == shared_lane("org/a", "org/b") assert result.report.identity("org/a").chosen.monthly_cost == pytest.approx(L4_HOURLY * MONTH) + assert result.report.identity("org/a").chosen.co_residents == ("org/b",) assert result.report.come_alive_levers == () loosened = planned({**bounds, "org/a": permitted}, receipts, boots, current=result.plan) - assert loosened.report.identity("org/a").chosen.warmth is Warmth.COLD_WITH_BOUND - assert loosened.report.monthly_cost - result.report.monthly_cost == pytest.approx(cold_cost()) + assert loosened.plan == result.plan + assert loosened.report.identity("org/a").chosen.warmth is Warmth.PINNED + assert loosened.report.monthly_cost == pytest.approx(result.report.monthly_cost) @pytest.mark.parametrize(("bound", "seconds", "permitted"), [(0, 95.2, 96), (60, 240.0, 240)]) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py new file mode 100644 index 000000000..2449df044 --- /dev/null +++ b/packages/sie_placement/tests/test_packing.py @@ -0,0 +1,694 @@ +from __future__ import annotations + +import random +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import pytest +from sie_placement.planner import PlannerParameters, plan +from sie_placement.report import IdentityStatus, Measurement, RequestReason, UnplacedReason, render +from sie_placement.schema import ( + BootEvidence, + BootOutcome, + Cell, + ContainerShape, + Demand, + DemandHour, + DemandSeries, + Envelope, + IdentityServiceLevel, + MachineProfilePrice, + Operation, + Percentiles, + Placement, + Plan, + PlanIdentity, + PlanLane, + PriceList, + Receipt, + RequestShape, + ServiceLevel, + ServiceLevels, + SourceRef, + Throughput, + Warmth, +) + +START = datetime(2026, 11, 2, tzinfo=UTC) +MONTH = 730.0 +L4 = Cell(machine_profile="l4", gpu_product="NVIDIA-L4") +L40S = Cell(machine_profile="l40s", gpu_product="NVIDIA-L40S") +A100 = Cell(machine_profile="a100-80gb", gpu_product="NVIDIA-A100-SXM4-80GB") +RATES = {L4: 1.0, L40S: 2.0, A100: 3.0} +CARD_MIB = {L4: 23034, L40S: 46068, A100: 81920} +LIMIT = 23034 * 0.8 +SHAPE = ContainerShape(gpu_count=1, cpu_cores=4, memory_mib=16384) +PAIR = ContainerShape(gpu_count=2, cpu_cores=8, memory_mib=32768) +HOST = "org/host" +GUEST = "org/guest" + + +def prices(*, cards=True) -> PriceList: + return PriceList( + currency="USD", + machine_profiles=tuple( + MachineProfilePrice( + name=cell.machine_profile, + gpu_product=cell.gpu_product, + accelerator_per_hour=rate, + core_per_hour=0.0, + memory_gib_per_hour=0.0, + gpu_memory_mib=CARD_MIB[cell] if cards else None, + ) + for cell, rate in RATES.items() + ), + placements=(Placement(name="test", region_multiplier=1),), + ) + + +def receipt( + identity, + cell=L4, + *, + beside=(), + footprint=None, + working=None, + p99=100.0, + rps=10.0, + shape=SHAPE, + concurrency=8, +) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=shape, + concurrency=concurrency, + request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), + throughput=Throughput(requests_per_second=rps), + latency=Percentiles(p99_ms=p99), + co_residents=tuple(beside), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef( + uri=f"receipts/{identity}/{cell.machine_profile}/{shape.gpu_count}/{concurrency}/{'+'.join(beside)}/{p99}" + ), + ) + + +def generation_receipt(identity, cell=L4, *, footprint=None, working=None) -> Receipt: + return Receipt( + identity=identity, + cell=cell, + container_shape=SHAPE, + concurrency=8, + request_shape=RequestShape(operation=Operation.GENERATE, items_per_request=1, output_tokens=128), + throughput=Throughput(requests_per_second=2.0, output_tokens_per_second=256.0), + ttft=Percentiles(p99_ms=500.0), + itl=Percentiles(p99_ms=40.0), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef(uri=f"receipts/{identity}/{cell.machine_profile}/generate"), + ) + + +def load(identity, cell=L4, seconds=20.0, *, footprint=None, outcome=BootOutcome.SUCCEEDED) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=SHAPE, + warmth=Warmth.LAZY_RESIDENT, + outcome=outcome, + boot_seconds=seconds, + footprint_mib=footprint, + source=SourceRef(uri=f"loads/{identity}/{cell.machine_profile}"), + ) + + +def cold(identity, cell=L4, seconds=30.0) -> BootEvidence: + return BootEvidence( + identity=identity, + cell=cell, + container_shape=SHAPE, + warmth=Warmth.COLD_WITH_BOUND, + outcome=BootOutcome.SUCCEEDED, + boot_seconds=seconds, + source=SourceRef(uri=f"boots/{identity}/{cell.machine_profile}"), + ) + + +def levels(**bounds) -> ServiceLevels: + return ServiceLevels( + identities=tuple( + IdentityServiceLevel(identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=250)) + for identity, bound in sorted(bounds.items()) + ) + ) + + +def generation_level(identity, bound) -> IdentityServiceLevel: + return IdentityServiceLevel( + identity=identity, level=ServiceLevel(come_alive_p95_s=bound, ttft_p99_ms=1000, itl_p99_ms=80) + ) + + +def demand(hourly=None) -> Demand: + return Demand( + start=START, + end=START + timedelta(hours=24), + series=tuple( + DemandSeries( + identity=identity, + operation=Operation.ENCODE, + hours=tuple( + DemandHour(start=START + timedelta(hours=hour), requests=count) + for hour, count in sorted(counts.items()) + ), + ) + for identity, counts in sorted((hourly or {}).items()) + ), + ) + + +def run(service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, **overrides): + parameters = PlannerParameters( + **({"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides) + ) + return plan(service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters) + + +def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: + return PlanLane( + name=name, + queue_pool="default", + bundle="default", + cell=cell, + container_shape=shape, + concurrency=8, + max_containers=4, + envelope=Envelope(floor_min=floor, floor_max=floor), + ) + + +def plan_of(lanes, residents) -> Plan: + return Plan( + placement="test", + lanes=tuple(lanes), + identities=tuple( + PlanIdentity(identity=identity, lane=name, warmth=warmth) + for identity, (name, warmth) in sorted(residents.items()) + ), + ) + + +def placements(result) -> dict[str, tuple[str, Warmth]]: + return {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} + + +def rendered(result) -> tuple[str, str]: + return render(result.plan.to_dict()), render(result.report.to_dict()) + + +def host_and_guest(guest_footprint=4000, guest_working=1500, *, guest_receipt=True): + receipts = [receipt(HOST, footprint=9000, working=2000)] + if guest_receipt: + receipts.append(receipt(GUEST, footprint=guest_footprint, working=guest_working)) + return receipts + + +def test_a_lazy_resident_joins_a_lane_formed_in_the_same_run_when_its_memory_fits(): + result = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)]) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert result.report.monthly_cost == pytest.approx(MONTH) + chosen = result.report.identity(GUEST).chosen + assert (chosen.lane, chosen.warmth, chosen.come_alive_s, chosen.monthly_cost) == ( + "org-host-l4", + Warmth.LAZY_RESIDENT, + 20.0, + 0, + ) + (planned,) = result.report.lanes + assert (planned.memory_mib, planned.memory_limit_mib) == (9000 + 4000 + 2000, pytest.approx(LIMIT)) + assert planned.to_dict()["memoryMiB"] == 15000 + again = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_lane_that_cannot_hold_the_resident_with_its_working_memory_hosts_none(): + result = run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(guest_footprint=8000), [load(GUEST)]) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] == [] + + +def test_lazy_residents_add_their_largest_footprint_and_the_largest_working_memory(): + receipts = [ + receipt(HOST, footprint=9000, working=2000), + receipt("org/g1", footprint=4000, working=1000), + receipt("org/g2", footprint=3000, working=3500), + ] + boots = [load("org/g1"), load("org/g2")] + result = run(levels(**{HOST: 0, "org/g1": 60, "org/g2": 60}), receipts, boots) + assert placements(result) == { + HOST: ("org-host-l4", Warmth.PINNED), + "org/g1": ("org-host-l4", Warmth.LAZY_RESIDENT), + "org/g2": ("org-host-l4", Warmth.LAZY_RESIDENT), + } + (planned,) = result.report.lanes + assert planned.memory_mib == 9000 + 4000 + 3500 + receipts[2] = receipt("org/g2", footprint=3000, working=6000) + result = run(levels(**{HOST: 0, "org/g1": 60, "org/g2": 60}), receipts, boots) + assert placements(result)["org/g2"] == ("org-g2-l4", Warmth.PINNED) + assert {lane.name: lane.memory_mib for lane in result.report.lanes}["org-host-l4"] == 15000 + + +def test_a_packing_without_footprints_is_a_measurement_request_with_its_saving_bound(): + receipts = [receipt(HOST, footprint=9000, working=2000), receipt(GUEST)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)]) + assert placements(result)[GUEST] == ("org-guest-l4", Warmth.PINNED) + requests = [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] + assert [(request.identity, request.warmth, request.measure, request.co_residents) for request in requests] == [ + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT, Measurement.FOOTPRINT), (HOST,)), + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.FOOTPRINT,), ()), + ] + for request in requests: + assert (request.reason, request.cost_bound, request.concurrency) == (RequestReason.UNMEASURED, 0, 8) + assert request.saving_bound == pytest.approx(MONTH) + assert requests[1].to_dict()["measure"] == ["footprint"] + + +def test_a_packing_without_a_load_asks_for_the_load_and_the_footprints_it_lacks(): + receipts = [receipt(HOST), receipt(GUEST, footprint=4000, working=1500)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts) + lazy = [ + request + for request in result.report.measurement_requests + if request.lane == "org-host-l4" and request.warmth is Warmth.LAZY_RESIDENT + ] + assert [request.measure for request in lazy] == [(Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)] + + +@pytest.mark.parametrize( + ("price_list", "shape"), [(prices(cards=False), SHAPE), (prices(), PAIR)], ids=["no-card-memory", "two-cards"] +) +def test_a_lane_whose_card_memory_is_unknown_or_whose_container_holds_several_cards_packs_nothing(price_list, shape): + receipts = [ + receipt(HOST, footprint=9000, working=2000, shape=shape), + receipt(GUEST, footprint=4000, working=1500, shape=shape), + ] + boots = [replace(load(GUEST), container_shape=shape)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, boots, price_list=price_list) + assert placements(result)[GUEST] == ("org-guest-l4", Warmth.PINNED) + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + + +def pair_receipts(*, a_p99=180.0, b_p99=200.0): + return [ + receipt("org/a", footprint=6000, working=1000), + receipt("org/b", footprint=5000, working=1000), + receipt("org/a", beside=["org/b"], footprint=6000, working=1500, p99=a_p99, rps=8.0), + receipt("org/b", beside=["org/a"], footprint=5000, working=1500, p99=b_p99, rps=8.0), + ] + + +def test_two_never_cold_identities_share_a_card_on_their_co_residency_receipt(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts()) + assert placements(result) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert result.report.monthly_cost == pytest.approx(MONTH) + for identity, beside in (("org/a", ("org/b",)), ("org/b", ("org/a",))): + report = result.report.identity(identity) + assert (report.lane, report.chosen.co_residents) == ("org-b-l4", beside) + assert report.chosen.to_dict()["coResidents"] == list(beside) + assert result.report.identity("org/a").chosen.lane == "org-b-l4" + (planned,) = result.report.lanes + assert planned.memory_mib == 6000 + 5000 + 1500 + again = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.identity("org/a").chosen.co_residents == ("org/b",) + + +def test_a_co_residency_receipt_over_a_warm_bound_keeps_the_identities_apart(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(b_p99=300.0)) + assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert all(request.lane is None for request in result.report.measurement_requests) + + +def test_a_co_pinned_resident_is_judged_by_its_co_residency_receipt(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + kept = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current) + assert kept.plan == current + assert kept.report.identity("org/a").chosen.co_residents == ("org/b",) + split = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(a_p99=300.0), current=current) + assert placements(split) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)} + assert split.report.identity("org/b").chosen.co_residents == () + again = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(a_p99=300.0), current=split.plan) + assert again.report.unchanged + + +def test_a_current_co_pinned_lane_without_its_co_residency_receipt_is_kept_and_asks_for_it(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts()[:2], current=current) + assert result.plan == current + for identity, beside in (("org/a", ("org/b",)), ("org/b", ("org/a",))): + report = result.report.identity(identity) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + (request,) = [request for request in result.report.measurement_requests if request.identity == identity] + assert (request.measure, request.lane, request.co_residents, request.saving_bound) == ( + (Measurement.CO_RESIDENCY_RECEIPT,), + "shared", + beside, + None, + ) + + +def test_a_co_residency_receipt_beside_more_residents_stands_for_fewer(): + current = plan_of( + [lane("shared")], + {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED), "org/c": ("shared", Warmth.PINNED)}, + ) + receipts = [ + receipt("org/a", beside=["org/b", "org/c"], p99=150.0), + receipt("org/b", beside=["org/a", "org/c"], p99=150.0), + receipt("org/c", beside=["org/a", "org/b"], p99=300.0), + receipt("org/c"), + ] + service_levels = levels(**{"org/a": 0, "org/b": 0, "org/c": 600}) + result = run(service_levels, receipts, [cold("org/c")], current=current) + assert placements(result) == { + "org/a": ("shared", Warmth.PINNED), + "org/b": ("shared", Warmth.PINNED), + "org/c": ("org-c-l4", Warmth.COLD_WITH_BOUND), + } + assert result.report.identity("org/a").status is IdentityStatus.PLACED + assert result.report.identity("org/a").chosen.co_residents == ("org/b", "org/c") + again = run(service_levels, receipts, [cold("org/c")], current=result.plan) + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + + +def test_generation_identities_are_never_packed(): + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 0), + IdentityServiceLevel(identity=GUEST, level=ServiceLevel(come_alive_p95_s=60, warm_p99_ms=250)), + ) + ) + receipts = [ + generation_receipt("org/llm", footprint=9000, working=2000), + receipt(GUEST, footprint=2000, working=500), + ] + result = run(service_levels, receipts, [load(GUEST)]) + assert placements(result) == {"org/llm": ("org-llm-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 600), + IdentityServiceLevel(identity=HOST, level=ServiceLevel(come_alive_p95_s=0, warm_p99_ms=250)), + ) + ) + receipts = [generation_receipt("org/llm", footprint=2000, working=500), receipt(HOST, footprint=9000, working=2000)] + result = run(service_levels, receipts, [load("org/llm")]) + assert placements(result)["org/llm"] == ("org-llm-l4", Warmth.PINNED) + + +def test_a_generation_identity_on_a_current_lane_hosts_no_lazy_resident(): + current = plan_of([lane("own"), lane("llm")], {"org/llm": ("llm", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + service_levels = ServiceLevels( + identities=( + generation_level("org/llm", 0), + IdentityServiceLevel(identity=GUEST, level=ServiceLevel(come_alive_p95_s=60, warm_p99_ms=250)), + ) + ) + receipts = [ + generation_receipt("org/llm", footprint=9000, working=2000), + receipt(GUEST, footprint=2000, working=500), + ] + result = run(service_levels, receipts, [load(GUEST)], current=current) + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + assert all(request.lane != "llm" for request in result.report.measurement_requests) + + +def test_packing_fills_the_lane_with_the_most_footprint_in_use_first(): + receipts = [ + receipt("org/h1", footprint=9000, working=1000), + receipt("org/h2", footprint=4000, working=1000), + receipt(GUEST, footprint=2000, working=1000), + ] + result = run(levels(**{"org/h1": 0, "org/h2": 0, GUEST: 60}), receipts, [load(GUEST)]) + assert placements(result) == { + "org/h1": ("org-h1-l4", Warmth.PINNED), + "org/h2": ("org-h2-l4", Warmth.PINNED), + GUEST: ("org-h1-l4", Warmth.LAZY_RESIDENT), + } + requests = [(request.identity, request.lane, request.warmth) for request in result.report.measurement_requests] + assert ("org/h2", "org-h1-l4", Warmth.PINNED) in requests + + +def test_packing_tries_lanes_a_never_cold_identity_pins_before_the_others(): + receipts = [ + receipt("org/a-warm", footprint=9000, working=1000), + receipt("org/b-never-cold", footprint=4000, working=1000), + receipt(GUEST, footprint=2000, working=1000), + ] + service_levels = levels(**{"org/a-warm": 60, "org/b-never-cold": 0, GUEST: 60}) + result = run(service_levels, receipts, [load(GUEST)]) + assert placements(result)[GUEST] == ("org-b-never-cold-l4", Warmth.LAZY_RESIDENT) + + +def test_a_host_that_can_itself_ride_a_never_cold_lane_takes_its_guests_along_in_one_run(): + receipts = [ + receipt("org/m1", footprint=5000, working=2500, rps=1.0), + receipt("org/m2", footprint=5000, working=500), + receipt("org/m3", footprint=9000, working=2500, rps=1.0), + ] + boots = [load("org/m1", seconds=45.0), load("org/m2", seconds=5.0), load("org/m3", seconds=45.0)] + service_levels = levels(**{"org/m1": 600, "org/m2": 0, "org/m3": 60}) + result = run(service_levels, receipts, boots) + assert placements(result) == { + "org/m1": ("org-m2-l4", Warmth.LAZY_RESIDENT), + "org/m2": ("org-m2-l4", Warmth.PINNED), + "org/m3": ("org-m2-l4", Warmth.LAZY_RESIDENT), + } + assert result.report.monthly_cost == pytest.approx(MONTH) + (planned,) = result.report.lanes + assert planned.memory_mib == 5000 + 9000 + 2500 + again = run(service_levels, receipts, boots, current=result.plan) + assert again.report.unchanged + + +@pytest.mark.parametrize(("footprint", "measure"), [(4000, None), (8000, ()), (None, (Measurement.FOOTPRINT,))]) +def test_the_lazy_resident_selection_on_a_current_lane_requires_the_memory_check(footprint, measure): + current = plan_of([lane("own"), lane("shared")], {HOST: ("shared", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + receipts = host_and_guest(guest_footprint=footprint) + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)], current=current) + requests = [ + request.measure + for request in result.report.measurement_requests + if request.lane == "shared" and request.warmth is Warmth.LAZY_RESIDENT + ] + if measure is None: + assert placements(result)[GUEST] == ("shared", Warmth.LAZY_RESIDENT) + assert requests == [] + else: + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + assert requests == ([measure] if measure else []) + + +def test_an_identity_whose_neighbours_departure_strips_its_co_residency_evidence_is_decided_again_in_the_run(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + receipts = [ + receipt("org/a", beside=["org/b"], p99=100.0), + receipt("org/b", beside=["org/a"], p99=300.0), + receipt("org/a", p99=300.0), + receipt("org/a", cell=A100), + receipt("org/b", cell=A100), + ] + service_levels = levels(**{"org/a": 0, "org/b": 0}) + result = run(service_levels, receipts, current=current) + assert placements(result) == { + "org/a": ("org-a-a100-80gb", Warmth.PINNED), + "org/b": ("org-b-a100-80gb", Warmth.PINNED), + } + again = run(service_levels, receipts, current=result.plan) + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + + +def test_packing_visits_identities_in_decreasing_floor_cost(): + current = plan_of( + [lane("host"), lane("double", floor=2)], + {HOST: ("host", Warmth.PINNED), "org/z": ("double", Warmth.PINNED)}, + ) + receipts = [ + receipt(HOST, footprint=9000, working=1000), + receipt("org/y", footprint=5000, working=1000), + receipt("org/z", footprint=5000, working=1000), + receipt(HOST, beside=["org/y"], footprint=9000, working=1000), + receipt("org/y", beside=[HOST], footprint=5000, working=1000), + receipt(HOST, beside=["org/z"], footprint=9000, working=1000), + receipt("org/z", beside=[HOST], footprint=5000, working=1000), + ] + result = run(levels(**{HOST: 0, "org/y": 0, "org/z": 0}), receipts, current=current) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + "org/y": ("org-y-l4", Warmth.PINNED), + "org/z": ("host", Warmth.PINNED), + } + assert result.report.monthly_cost == pytest.approx(2 * MONTH) + assert all(request.identity != "org/y" or request.lane is None for request in result.report.measurement_requests) + again = run(levels(**{HOST: 0, "org/y": 0, "org/z": 0}), receipts, current=result.plan) + assert again.report.unchanged + + +def test_a_pinned_identity_stays_beside_a_neighbour_the_lane_stays_for(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + receipts = [*pair_receipts(), receipt("org/b", cell=A100)] + result = run(levels(**{"org/a": 60, "org/b": 0}), receipts, [cold("org/a")], current=current) + assert result.plan == current + assert result.report.monthly_cost == pytest.approx(MONTH) + report = result.report.identity("org/a") + assert report.chosen.warmth is Warmth.PINNED + both = run(levels(**{"org/a": 60, "org/b": 60}), receipts, [cold("org/a"), cold("org/b")], current=current) + assert placements(both) == { + "org/a": ("org-a-l4", Warmth.COLD_WITH_BOUND), + "org/b": ("org-b-l4", Warmth.COLD_WITH_BOUND), + } + assert both.report.monthly_cost == 0 + + +def test_identities_that_leave_a_lane_together_return_when_sharing_it_costs_less(): + current = plan_of( + [lane("shared", cell=A100)], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)} + ) + receipts = [ + receipt("org/a", cell=A100, beside=["org/b"]), + receipt("org/b", cell=A100, beside=["org/a"]), + receipt("org/a", cell=L40S), + receipt("org/b", cell=L40S), + ] + result = run(levels(**{"org/a": 0, "org/b": 0}), receipts, current=current) + assert result.plan == current + assert result.report.monthly_cost == pytest.approx(3 * MONTH) + alone = plan_of([lane("shared", cell=A100)], {"org/a": ("shared", Warmth.PINNED)}) + single = run(levels(**{"org/a": 0}), receipts, current=alone) + assert placements(single) == {"org/a": ("org-a-l40s", Warmth.PINNED)} + + +IDENTITIES = ("org/m0", "org/m1", "org/m2", "org/m3", "org/m4") + + +@pytest.mark.parametrize("seed", range(150)) +def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + identities = list(IDENTITIES[: rng.randint(2, 5)]) + generation = {identity for identity in identities if rng.random() < 0.15} + entries = [] + for identity in identities: + bound = rng.choice([0, 0, 30, 60, 600]) + if identity in generation: + entries.append(generation_level(identity, bound)) + else: + entries.append( + IdentityServiceLevel( + identity=identity, level=ServiceLevel(come_alive_p95_s=bound, warm_p99_ms=rng.choice([150, 250])) + ) + ) + service_levels = ServiceLevels(identities=tuple(entries)) + footprints = {identity: rng.choice([None, 2000, 5000, 9000]) for identity in identities} + working = {identity: rng.choice([None, 500, 2500]) for identity in identities} + receipts = [] + for identity in identities: + for cell in (L4, A100): + if rng.random() < 0.8: + if identity in generation: + receipts.append(generation_receipt(identity, cell, footprint=footprints[identity], working=500)) + else: + receipts.append( + receipt( + identity, + cell, + footprint=footprints[identity], + working=working[identity], + p99=rng.choice([100.0, 200.0]), + rps=rng.choice([1.0, 10.0]), + ) + ) + plain = [identity for identity in identities if identity not in generation] + for _ in range(rng.randint(0, 4)): + group = rng.sample(plain, min(len(plain), rng.randint(2, 3))) if len(plain) >= 2 else [] + for identity in group: + receipts.append( + receipt( + identity, + beside=sorted(set(group) - {identity}), + footprint=footprints[identity], + working=rng.choice([500, 2500]), + p99=rng.choice([150.0, 300.0]), + rps=rng.choice([1.0, 10.0]), + ) + ) + boots = [load(identity, seconds=rng.choice([5.0, 45.0, 200.0])) for identity in identities if rng.random() < 0.7] + boots.extend(cold(identity, seconds=rng.choice([20.0, 300.0])) for identity in identities if rng.random() < 0.3) + hourly = { + identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} + for identity in identities + } + current = None + if rng.random() < 0.5: + first = run(service_levels, receipts, boots, hourly=hourly) + current = first.plan + result = run(service_levels, receipts, boots, hourly=hourly, current=current) + assert Plan.from_dict(result.plan.to_dict()) == result.plan + levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} + residents: dict[str, list[PlanIdentity]] = {} + for entry in result.plan.identities: + residents.setdefault(entry.lane, []).append(entry) + before = {} if current is None else {planned.name: planned for planned in current.lanes} + reports = {lane_report.name: lane_report for lane_report in result.report.lanes} + for planned in result.plan.lanes: + hosted = residents[planned.name] + if any(entry.identity in generation for entry in hosted): + assert len(hosted) == 1 + if len(hosted) < 2: + continue + pinned = {entry.identity for entry in hosted if entry.warmth is Warmth.PINNED} + formed = planned.name not in before or { + entry.identity for entry in current.identities if entry.lane == planned.name + } != {entry.identity for entry in hosted} + if formed: + assert reports[planned.name].memory_mib is not None + assert reports[planned.name].memory_mib <= reports[planned.name].memory_limit_mib + for entry in hosted: + report = result.report.identity(entry.identity) + if report.chosen is None: + continue + level = levels_by_identity[entry.identity] + if level.warm_p99_ms is not None: + assert report.chosen.blocked_by == () + if entry.warmth is Warmth.PINNED and len(pinned) > 1: + assert set(report.chosen.co_residents) >= pinned - {entry.identity} + if entry.warmth is Warmth.LAZY_RESIDENT: + assert level.come_alive_p95_s != 0 + assert level.come_alive_p95_s is None or report.chosen.come_alive_s <= level.come_alive_p95_s + for request in result.report.measurement_requests: + assert request.saving_bound is None or request.saving_bound > 0 + again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + shuffled = list(receipts) + rng.shuffle(shuffled) + reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current) + assert rendered(reordered) == rendered(result) + for entry in service_levels.identities: + bound = entry.level.come_alive_p95_s + looser = ServiceLevels( + identities=tuple( + replace(other, level=replace(other.level, come_alive_p95_s=bound + 600)) + if other.identity == entry.identity + else other + for other in service_levels.identities + ) + ) + loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan) + assert loosened.report.monthly_cost <= result.report.monthly_cost + 0.01 diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 2bcbd0718..c4bf3886c 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -79,18 +79,26 @@ ) -def profile(cell: Cell, accelerator: float, core: float = 0.0) -> MachineProfilePrice: +CARD_MIB = {L4: 23034, A100: 81920} + + +def profile(cell: Cell, accelerator: float, core: float = 0.0, memory: int | None = None) -> MachineProfilePrice: return MachineProfilePrice( name=cell.machine_profile, gpu_product=cell.gpu_product, accelerator_per_hour=accelerator, core_per_hour=core, memory_gib_per_hour=0.0, + gpu_memory_mib=memory, ) -def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=()) -> PriceList: - profiles = [profile(cell, rate, core) for cell, rate in ((L4, l4), (A100, a100)) if rate is not None] +def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=(), cards=True) -> PriceList: + profiles = [ + profile(cell, rate, core, CARD_MIB[cell] if cards else None) + for cell, rate in ((L4, l4), (A100, a100)) + if rate is not None + ] return PriceList( currency="USD", machine_profiles=(*profiles, *extra), @@ -101,7 +109,20 @@ def price_list(*, l4=1.0, a100=3.0, core=0.0, multiplier=1.0, extra=()) -> Price L4_ONLY = price_list(a100=None) -def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, p99=100.0, uri=None) -> Receipt: +def receipt( + identity=IDENTITY, + cell=L4, + *, + shape=SHAPE, + concurrency=8, + rps=10.0, + p99=100.0, + uri=None, + co_residents=(), + footprint=None, + working=None, +) -> Receipt: + beside = "".join(f"+{name}" for name in co_residents) return Receipt( identity=identity, cell=cell, @@ -110,7 +131,12 @@ def receipt(identity=IDENTITY, cell=L4, *, shape=SHAPE, concurrency=8, rps=10.0, request_shape=RequestShape(operation=Operation.ENCODE, items_per_request=1), throughput=Throughput(requests_per_second=rps), latency=Percentiles(p99_ms=p99), - source=SourceRef(uri=uri or f"receipts/{identity}/{cell.machine_profile}-{concurrency}-{shape.memory_mib}"), + co_residents=tuple(co_residents), + footprint_mib=footprint, + working_memory_mib=working, + source=SourceRef( + uri=uri or f"receipts/{identity}/{cell.machine_profile}-{concurrency}-{shape.memory_mib}{beside}" + ), ) @@ -269,7 +295,7 @@ def resident_estate() -> Plan: ) -RESIDENT_RECEIPTS = [receipt(identity=ANCHOR), receipt()] +RESIDENT_RECEIPTS = [receipt(identity=ANCHOR, footprint=6000, working=2000), receipt(footprint=3000, working=1000)] LOADS = [boot(warmth=Warmth.LAZY_RESIDENT, seconds=20.0, uri="loads/example.json")] @@ -736,16 +762,17 @@ def test_the_next_cheaper_option_is_the_nearest_cheaper_one_and_names_its_bound( def test_a_next_cheaper_saving_is_what_loosening_its_bounds_saves_the_plan(others, saving): service_levels = levels(*((name, ServiceLevel(come_alive_p95_s=0)) for name in (IDENTITY, *others))) current = single_lane_plan(others=others) - result = run(service_levels, [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=current) + receipts = [receipt(), *([receipt(co_residents=others)] if others else [])] + result = run(service_levels, receipts, [boot(seconds=30.0)], prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) assert (report.lane, report.chosen.warmth) == ("shared", Warmth.PINNED) assert (report.next_cheaper.warmth, report.next_cheaper.monthly_cost) == (Warmth.COLD_WITH_BOUND, 0.0) assert report.chosen.monthly_cost - report.next_cheaper.monthly_cost == pytest.approx(MONTH) assert report.next_cheaper.saving == pytest.approx(saving) again = run( - loosened(service_levels, IDENTITY, 30), [receipt()], [boot(seconds=30.0)], prices=L4_ONLY, current=result.plan + loosened(service_levels, IDENTITY, 30), receipts, [boot(seconds=30.0)], prices=L4_ONLY, current=result.plan ) - assert again.report.identity(IDENTITY).chosen.warmth is Warmth.COLD_WITH_BOUND + assert again.report.identity(IDENTITY).chosen.warmth is (Warmth.PINNED if others else Warmth.COLD_WITH_BOUND) assert result.report.monthly_cost - again.report.monthly_cost == pytest.approx(saving) @@ -1220,21 +1247,19 @@ def test_a_load_over_the_bound_is_a_come_alive_lever(): assert lever.saving_under_bound == pytest.approx(MONTH) -def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_from_the_next_run(): +def test_a_lane_a_run_creates_for_a_never_cold_identity_hosts_lazy_residents_in_the_same_run(): current = plan_of([plan_lane("own")], {IDENTITY: ("own", Warmth.PINNED)}) first = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) assert [(entry.identity, entry.lane, entry.warmth) for entry in first.plan.identities] == [ - (ANCHOR, "org-anchor-l4", Warmth.PINNED), - (IDENTITY, "own", Warmth.PINNED), - ] - second = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=first.plan) - assert [(entry.identity, entry.lane, entry.warmth) for entry in second.plan.identities] == [ (ANCHOR, "org-anchor-l4", Warmth.PINNED), (IDENTITY, "org-anchor-l4", Warmth.LAZY_RESIDENT), ] - assert first.report.monthly_cost - second.report.monthly_cost == pytest.approx(MONTH) - third = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=second.plan) - assert third.report.unchanged + assert first.report.monthly_cost == pytest.approx(MONTH) + chosen = first.report.identity(IDENTITY).chosen + assert (chosen.lane, chosen.come_alive_s, chosen.monthly_cost) == ("org-anchor-l4", 20.0, 0) + second = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=first.plan) + assert second.plan == first.plan + assert second.report.unchanged def test_a_lazy_resident_never_joins_a_lane_its_never_cold_neighbour_leaves(): @@ -1250,7 +1275,11 @@ def test_a_lazy_resident_never_joins_a_lane_its_never_cold_neighbour_leaves(): (IDENTITY, "own", Warmth.PINNED), ] assert result.report.monthly_cost == pytest.approx(2 * MONTH) - assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + assert all(request.lane != "shared" for request in result.report.measurement_requests) + lazy = [request for request in result.report.measurement_requests if request.warmth is Warmth.LAZY_RESIDENT] + assert [(request.lane, request.measure) for request in lazy] == [ + ("org-anchor-l4", (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)) + ] again = run(resident_levels(), receipts, loads, current=result.plan) assert again.report.unchanged @@ -1263,7 +1292,7 @@ def test_a_lazy_resident_whose_lane_keeps_no_never_cold_neighbour_pays_the_lane_ ("org/warm", ServiceLevel(come_alive_p95_s=120, warm_p99_ms=ANY_P99_MS)), (IDENTITY, ServiceLevel(come_alive_p95_s=60, warm_p99_ms=ANY_P99_MS)), ) - receipts = [receipt(identity="org/warm"), receipt()] + receipts = [receipt(identity="org/warm", footprint=6000, working=2000), receipt(footprint=3000, working=1000)] boots = [boot(identity="org/warm"), *LOADS, boot(seconds=30.0)] hourly = {"org/warm": {0: 1}, IDENTITY: {0: 1}} result = run(service_levels, receipts, boots, prices=L4_ONLY, hourly=hourly, current=current) @@ -1280,7 +1309,8 @@ def test_a_lazy_resident_whose_lane_keeps_no_never_cold_neighbour_pays_the_lane_ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_than_the_lane_floor(): current = plan_of([plan_lane("shared")], {IDENTITY: ("shared", Warmth.LAZY_RESIDENT)}) - result = run(level(come_alive_p95_s=60), [receipt()], LOADS, prices=L4_ONLY, current=current) + receipts = [receipt(footprint=3000, working=1000)] + result = run(level(come_alive_p95_s=60), receipts, LOADS, prices=L4_ONLY, current=current) assert result.plan == current assert result.report.unchanged report = result.report.identity(IDENTITY) @@ -1293,37 +1323,55 @@ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_th assert result.report.monthly_cost == pytest.approx(MONTH) -@pytest.mark.parametrize( - ("shared", "anchor_bound"), - [ - (plan_lane("shared"), 30), - (plan_lane("shared", bundle="sglang"), 0), - (plan_lane("shared", queue_pool="batch"), 0), - ], -) -def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_lane_of_its_pool_and_bundle(shared, anchor_bound): +@pytest.mark.parametrize("shared", [plan_lane("shared", bundle="sglang"), plan_lane("shared", queue_pool="batch")]) +def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): current = plan_of([plan_lane("own"), shared], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}) - result = run(resident_levels(anchor_bound=anchor_bound), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities} == { + ANCHOR: "org-anchor-l4", + IDENTITY: "org-anchor-l4", + } + assert all(entry.lane != "shared" for entry in result.plan.identities) + assert all(request.lane != "shared" for request in result.report.measurement_requests) + + +def test_a_current_lane_hosts_lazy_residents_beside_a_never_cold_neighbour_and_a_plan_lane_beside_any_pinned_one(): + current = plan_of( + [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} + ) + result = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) - assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert (report.lane, report.chosen.warmth, report.chosen.lane) == ("shared", Warmth.LAZY_RESIDENT, "shared") + assert result.report.monthly_cost == pytest.approx(MONTH) assert all(lever.warmth is not Warmth.LAZY_RESIDENT for lever in result.report.come_alive_levers) - assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + again = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=result.plan) + assert again.report.unchanged @pytest.mark.parametrize( - ("bound", "expected"), [(60, [(Warmth.COLD_WITH_BOUND, None), (Warmth.LAZY_RESIDENT, "shared")]), (0, [])] + ("bound", "expected"), + [ + ( + 60, + [ + (Warmth.COLD_WITH_BOUND, None, (Measurement.BOOT_EVIDENCE,)), + (Warmth.LAZY_RESIDENT, "shared", (Measurement.BOOT_EVIDENCE,)), + (Warmth.PINNED, "shared", (Measurement.CO_RESIDENCY_RECEIPT,)), + ], + ), + (0, [(Warmth.PINNED, "shared", (Measurement.CO_RESIDENCY_RECEIPT,))]), + ], ) def test_a_warm_lane_without_a_load_row_asks_for_one_at_the_saving_it_unlocks(bound, expected): result = run(resident_levels(bound=bound), RESIDENT_RECEIPTS, prices=L4_ONLY, current=resident_estate()) requests = [request for request in result.report.measurement_requests if request.identity == IDENTITY] - assert [(request.warmth, request.lane) for request in requests] == expected + assert [(request.warmth, request.lane, request.measure) for request in requests] == expected for request in requests: - assert (request.measure, request.reason, request.cost_bound) == ( - (Measurement.BOOT_EVIDENCE,), - RequestReason.UNMEASURED, - 0, - ) + assert (request.reason, request.cost_bound) == (RequestReason.UNMEASURED, 0) assert request.saving_bound == pytest.approx(MONTH) + (copin,) = [request for request in requests if request.warmth is Warmth.PINNED] + assert copin.co_residents == (ANCHOR,) + assert copin.to_dict()["coResidents"] == [ANCHOR] def test_a_current_placement_on_a_shared_lane_stays_beside_its_kept_neighbour(): @@ -1595,15 +1643,17 @@ def test_the_estate_report_states_its_findings(): if request.lane is not None ] assert on_lanes == [ - ("intfloat/e5-base-v2", (Measurement.RECEIPT,), Warmth.PINNED, "default-l4-default", None), + ("intfloat/e5-base-v2", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED, "default-l4-default", None), ( "BAAI/bge-m3:sparse", - (Measurement.BOOT_EVIDENCE,), + (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT), Warmth.LAZY_RESIDENT, "default-l4-default", pytest.approx(sparse.chosen.monthly_cost), ), ] + assert dense.chosen.co_residents == ("intfloat/e5-base-v2",) + assert dense.chosen.receipt.uri == "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" assert [change.identity for change in report.identity_changes] == ["BAAI/bge-m3:sparse", "Qwen/Qwen3-Embedding-4B"] assert report.monthly_cost == pytest.approx(sum(lane.monthly_cost for lane in report.lanes)) shared = {lane.name: lane for lane in report.lanes}["default-l4-default"] @@ -1646,8 +1696,9 @@ def test_an_estate_identity_beside_a_kept_neighbour_offers_no_lever_and_no_savin for bound in (9, 95): service_levels = loosened(inputs["service_levels"], "BAAI/bge-m3", bound) again = plan(**{**inputs, "service_levels": service_levels, "current_plan": result.plan}) - assert again.report.identity("BAAI/bge-m3").chosen.warmth in (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) - assert again.report.monthly_cost > result.report.monthly_cost + assert again.report.identity("BAAI/bge-m3").chosen.warmth is Warmth.PINNED + assert again.plan == result.plan + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) def test_an_estate_next_cheaper_saving_is_what_planning_again_with_its_bounds_loosened_saves(): diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index 841c8eead..e63d22c61 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -452,6 +452,79 @@ def test_capacity_built_in_code_names_type_ceilings_by_gpu_product(): assert caught.value.path == "gpuProduct" +def test_receipts_record_the_memory_a_model_holds_and_its_co_residents(): + data = load("receipt-encode.json") + data["coResidents"] = ["intfloat/e5-base-v2", "BAAI/bge-m3:sparse"] + data["footprintMiB"] = 2310 + data["workingMemoryMiB"] = 0 + receipt = Receipt.from_dict(data) + assert (receipt.co_residents, receipt.footprint_mib, receipt.working_memory_mib) == ( + ("intfloat/e5-base-v2", "BAAI/bge-m3:sparse"), + 2310, + 0, + ) + assert receipt.to_dict() == data + assert list(receipt.to_dict())[:7] == [ + "kind", + "schemaVersion", + "identity", + "cell", + "containerShape", + "concurrency", + "coResidents", + ] + plain = Receipt.from_dict(load("receipt-encode.json")) + assert (plain.co_residents, plain.footprint_mib, plain.working_memory_mib) == ((), None, None) + assert "coResidents" not in plain.to_dict() + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("coResidents", ["BAAI/bge-m3"], "$.coResidents[0]: names the receipt's own identity 'BAAI/bge-m3'"), + ("coResidents", ["org/a", "org/a"], "$.coResidents[1]: duplicates 'org/a'"), + ("coResidents", ["org/a", ""], "$.coResidents[1]: must be a non-empty string without whitespace, got ''"), + ("coResidents", [7], "$.coResidents[0]: expected a string, got number 7"), + ("coResidents", "org/a", "$.coResidents: expected an array, got string 'org/a'"), + ("footprintMiB", 0, "$.footprintMiB: must be at least 1, got 0"), + ("footprintMiB", 1.5, "$.footprintMiB: expected an integer, got number 1.5"), + ("workingMemoryMiB", -1, "$.workingMemoryMiB: must be at least 0, got -1"), + ], +) +def test_receipt_memory_and_co_residents_are_validated(field, value, message): + data = load("receipt-encode.json") + data[field] = value + assert str(schema_error(Receipt.from_dict, data)) == message + + +def test_boot_evidence_records_the_footprint_of_a_load_that_succeeded(): + data = load("boot-evidence.json") + data["footprintMiB"] = 2290 + evidence = BootEvidence.from_dict(data) + assert evidence.footprint_mib == 2290 + assert evidence.to_dict() == data + data["footprintMiB"] = 0 + assert str(schema_error(BootEvidence.from_dict, data)) == "$.footprintMiB: must be at least 1, got 0" + data["footprintMiB"] = 2290 + data["outcome"] = "failed" + assert str(schema_error(BootEvidence.from_dict, data)) == ( + "$.footprintMiB: must be omitted for an attempt that did not succeed" + ) + + +def test_machine_profiles_may_state_the_memory_of_their_card(): + data = load("price-list.json") + data["machineProfiles"][0]["gpuMemoryMiB"] = 23034 + prices = PriceList.from_dict(data) + assert prices.machine_profiles[0].gpu_memory_mib == 23034 + assert prices.machine_profiles[1].gpu_memory_mib is None + assert prices.to_dict() == data + data["machineProfiles"][0]["gpuMemoryMiB"] = 0 + assert str(schema_error(PriceList.from_dict, data)) == ( + "$.machineProfiles[0].gpuMemoryMiB: must be at least 1, got 0" + ) + + def test_price_lists_validate_currency_rates_and_names(): data = load("price-list.json") data["currency"] = "usd" From 62481f7954e2e5d9d94e3e1d5a442b7011f00150 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 18:45:39 +0100 Subject: [PATCH 02/10] fix(placement): hold joined lanes to their memory and ask the right resident for evidence - Identities that join a lane in one run must fit it together. Each lazy join was checked only beside the residents the lane had before, so two joiners that fit alone could overfill the card. Joiners that do not fit together are admitted first fit in decreasing footprint and the rest are decided again without that lane. - A missing footprint or working memory is requested from the resident that lacks it, under its own identity and warmth at the lane, instead of from the joining identity. A kept lazy resident whose lane lacks memory evidence now asks for it, a profile without gpuMemoryMiB is named in the report as gpuMemoryUndeclared, and a repeated request is listed once. - Packing makes a lazy resident only on a lane where a never-cold identity is pinned, as the lazy-resident posture requires, and visits a lazy resident that is alone on its own lane like any other identity alone on its lane. - The unshared parameter names identities that never share a lane: they are never packed or made lazy residents, and no identity joins a lane they are on. - The random-estate property test also starts from current plans that are not the planner's own fixpoint, and a new property test checks that guests joining one host lane fit it together. --- packages/sie_placement/README.md | 83 +++-- .../src/sie_placement/planner.py | 327 +++++++++++++----- .../sie_placement/src/sie_placement/report.py | 2 + .../fixtures/planner/expected-report.json | 25 +- packages/sie_placement/tests/test_packing.py | 273 +++++++++++++-- packages/sie_placement/tests/test_planner.py | 35 +- 6 files changed, 589 insertions(+), 156 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 5ede7323d..29b5b6adc 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -155,8 +155,20 @@ than by a receipt of it alone: one whose `coResidents` include every other identity pinned on the lane, since a receipt measured beside more residents stands for fewer. Its warm bounds are checked and its demand is priced at that receipt. The planner never places a generation identity, one that declares -`ttftP99Ms` and `itlP99Ms` or has a generate receipt, beside another identity, -nor another identity on a lane that hosts one. +`ttftP99Ms` and `itlP99Ms` or has a generate receipt, or an identity that the +`unshared` parameter names, beside another identity, nor another identity on a +lane that hosts one. Such an identity keeps a current lane it already shares. + +Every lane that an identity joins in a run must fit all of its residents in +the plan that run writes, not only each joining identity beside the residents +it had before. Where the identities that join a lane do not fit it together, +they are admitted first fit in decreasing order of footprint, ties broken by +identity, and the others are decided again without that lane. + +Where a resident's footprint or working memory is not on file, the planner +asks that resident for it, under its own identity and warmth at the lane, in a +`footprint` measurement request. A lane on a profile without `gpuMemoryMiB` +hosts no identity beside another, and the report names that profile. ## Envelope @@ -207,6 +219,7 @@ choice, such as its own previous plan, reports no change. | `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | | `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | +| `unshared` | Identities that never share a lane: none is packed or made a lazy resident beside another identity, and no other identity joins a lane one of them is on. Empty by default | ### Selection @@ -255,9 +268,9 @@ document: it. The host must fit the identity beside the residents of its current lane, as [Several models on one GPU](#several-models-on-one-gpu) states; where a footprint or a working memory is not on file the option is a - measurement request for it. A lane with fallback cells, or one that hosts a - generation identity, hosts no lazy resident, and a generation identity is - never one. + measurement request for it, asked of each resident that lacks it. A lane + with fallback cells, or one that hosts a generation or `unshared` identity, + hosts no lazy resident, and neither identity is ever one. 5. A GPU product is unreliable when more than 5% of the cold attempts on it were `unplaced`, counting the environment's own attempts in place of borrowed ones. For an identity whose `comeAliveP95S` is at most @@ -316,22 +329,26 @@ plan total does not fall. 2. Identities that left a current lane return to it, one at a time and then together, wherever their return does not raise the plan total. An identity whose current placement is known to miss a bound does not return. -3. Packing. The planner visits every placed identity that is alone on its lane - and is not a generation identity, in decreasing order of its lane's floor - cost, ties broken by identity. For each it tries the other lanes of the plan - on its queue pool and bundle that keep a floor and host a pinned identity, - where it has a receipt at the lane's cell, container shape and concurrency: - first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, - since nothing moves that identity, then in decreasing order of the footprint - already on them, then by name. On the first lane where a move fits and lowers - the plan total by at least a cent, the identity joins it, as a lazy resident - when its bound admits its measured load onto the lane and selection did not - already offer it the lane, otherwise pinned, which needs every pinned - identity's co-residency receipt beside the others to meet its warm bounds. A - move that would lower the total but lacks a footprint, a working memory, a - load or a co-residency receipt is not made: it is a measurement request whose - `savingBound` is that fall. -4. Step 1 runs again for the lanes packing changed. +3. The identities that joined a lane are held to fitting it together, as + [Several models on one GPU](#several-models-on-one-gpu) states, and step 1 + runs again for every identity decided again. +4. Packing. The planner visits every placed identity that is alone on its lane + and is neither a generation nor an `unshared` identity, in decreasing order + of its lane's floor cost, ties broken by identity. For each it tries the + other lanes of the plan on its queue pool and bundle that keep a floor and + host a pinned identity, where it has a receipt at the lane's cell, container + shape and concurrency: first the lanes on which an identity whose + `comeAliveP95S` is `0` is pinned, since nothing moves that identity, then in + decreasing order of the footprint already on them, then by name. On the + first lane where a move fits and lowers the plan total by at least a cent, + the identity joins it, as a lazy resident when an identity whose + `comeAliveP95S` is `0` is pinned on the lane, its bound admits its measured + load onto the lane and selection did not already offer it the lane, + otherwise pinned, which needs every pinned identity's co-residency receipt + beside the others to meet its warm bounds. A move that would lower the total + but lacks a footprint, a working memory, a load or a co-residency receipt is + not made: it is a measurement request whose `savingBound` is that fall. +5. Steps 1 and 3 run again for the lanes packing changed. ### Pricing @@ -433,14 +450,19 @@ The report renders as a `placement-report` document: reason `failed`, or `unplaced` when no attempt got a container or the product's placement wait falls on an unplaced attempt, and a host lane without the identity's load asks for `lazy-resident` boot evidence on it, bounded by - what that option would cost, with `footprint` also asked for where a - resident's footprint or working memory is not on file. A packing that lacks - evidence asks at the lane it would join for `boot-evidence`, `footprint` or - `co-residency-receipt`, with the `coResidents` the receipt is to be measured - beside, bounded by the fall in the plan total the packing would bring. A kept - identity pinned beside others asks at its lane for a `co-residency-receipt` - beside them. Savings are measured against the choice, or against a kept - identity's current placement. A request that cannot save is dropped, and the + what that option would cost, with `footprint` asked of each resident whose + footprint or working memory is not on file, under that resident's identity + and warmth. A packing that lacks evidence asks at the lane it would join for + `boot-evidence`, `footprint` or `co-residency-receipt`, with the + `coResidents` the receipt is to be measured beside, bounded by the fall in + the plan total the packing would bring, and asks a resident there for its + own missing `footprint` the same way. A kept identity pinned beside others + asks at its lane for a `co-residency-receipt` beside them, and a kept lazy + resident asks each resident of its lane, itself included, for a missing + `footprint`. A request repeated for the same identity, cell, container + shape, concurrency, lane, measurements, warmth and `coResidents` is listed + once, at its highest rank. Savings are measured against the choice, or + against a kept identity's current placement. A request that cannot save is dropped, and the requests of an identity without a placement or a current one carry no bound and rank first. The evidence a kept current placement lacks is requested at its lane, named in `lane` with its `concurrency`, and also ranks first without @@ -473,6 +495,9 @@ The report renders as a `placement-report` document: plan claims that bound as met. - `warmBoundUndeclared`: every identity that has a cell and declares neither `warmP99Ms` nor `ttftP99Ms` and `itlP99Ms`, so keeps its current class. +- `gpuMemoryUndeclared`: every machine profile without `gpuMemoryMiB` on which + the planner weighed an identity beside another, so its lanes host no + identity beside another and a lane that already shares one is not checked. - `refusals`, and `changes`: the identities and lanes that differ from `current_plan`. diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 03bb6424d..14e3690b0 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -103,6 +103,7 @@ class PlannerParameters: queue_pool: str = "default" bundle: str = "default" bundles: Mapping[str, str] = field(default_factory=dict) + unshared: frozenset[str] = frozenset() def __post_init__(self) -> None: _require_name("placement", self.placement) @@ -118,6 +119,8 @@ def __post_init__(self) -> None: for identity, bundle in self.bundles.items(): _require_name("bundles", identity) _require_name(f"bundles[{identity!r}]", bundle) + for identity in self.unshared: + _require_name("unshared", identity) @property def active_floor(self) -> int: @@ -140,6 +143,8 @@ def to_dict(self) -> dict[str, object]: } if self.bundles: document["bundles"] = dict(sorted(self.bundles.items())) + if self.unshared: + document["unshared"] = sorted(self.unshared) return document @@ -331,6 +336,7 @@ class _Layout: type _CoKey = tuple[str, Cell, ContainerShape, int] +type _Packing = tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...], tuple[tuple[str, Warmth], ...]] @dataclass(frozen=True, slots=True, kw_only=True) @@ -581,6 +587,8 @@ def __init__( self.refused_profiles: set[tuple[str, str]] = set() self.savings: dict[tuple[str, tuple[tuple[str, int], ...]], float] = {} self.co_receipts: defaultdict[_CoKey, list[tuple[str, Receipt]]] = defaultdict(list) + self.excluded: set[tuple[str, str]] = set() + self.gpu_memory_undeclared: set[str] = set() self.receipts = self._accept_receipts(receipts) self.placements = _placement_index( record for record in boot_evidence if _refusal_reason(record.cell, self.prices) is None @@ -597,10 +605,11 @@ def __init__( self.working[key] = max(receipt.working_memory_mib, self.working.get(key, 0)) for record in accepted: self._footprint(record.identity, record.cell, record.footprint_mib) - self.generation = {identity for identity, level in self.levels.items() if level.is_generation} - self.generation.update( + self.solitary = {identity for identity, level in self.levels.items() if level.is_generation} + self.solitary.update( receipt.identity for receipt in measured if receipt.request_shape.operation is Operation.GENERATE ) + self.solitary.update(parameters.unshared) def _footprint(self, identity: str, cell: Cell, footprint: int | None) -> None: if footprint is not None: @@ -687,20 +696,17 @@ def co_receipt(self, identity: str, lane: PlanLane, beside: frozenset[str]) -> t default=None, ) - def hosts_generation(self, identities: Iterable[str]) -> bool: - return any(identity in self.generation for identity in identities) + def hosts_solitary(self, identities: Iterable[str]) -> bool: + return any(identity in self.solitary for identity in identities) def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> _Memory: pinned, lazy = tuple(sorted(pinned)), tuple(sorted(lazy)) price = self.prices.get(lane.cell.machine_profile) residents = (*pinned, *lazy) - if ( - not residents - or price is None - or price.gpu_memory_mib is None - or lane.container_shape.gpu_count != 1 - or lane.fallback_cells - ): + if not residents or price is None or lane.container_shape.gpu_count != 1 or lane.fallback_cells: + return _UNPACKABLE + if price.gpu_memory_mib is None: + self.gpu_memory_undeclared.add(price.name) return _UNPACKABLE limit = price.gpu_memory_mib * MEMORY_HEADROOM_FRACTION config = (lane.cell, lane.container_shape, lane.concurrency) @@ -716,6 +722,38 @@ def memory(self, lane: PlanLane, pinned: Iterable[str], lazy: Iterable[str]) -> ) return _Memory(fits=need <= limit, need=need, limit=limit) + def lane_memory(self, lane: PlanLane, residents: Mapping[str, Warmth]) -> _Memory: + return self.memory( + lane, + pinned=(identity for identity, warmth in residents.items() if warmth is Warmth.PINNED), + lazy=(identity for identity, warmth in residents.items() if warmth is Warmth.LAZY_RESIDENT), + ) + + def memory_gaps(self, lane: PlanLane, residents: Iterable[str]) -> tuple[str, ...]: + config = (lane.cell, lane.container_shape, lane.concurrency) + return tuple( + identity + for identity in sorted(set(residents)) + if (identity, lane.cell) not in self.footprints or (identity, *config) not in self.working + ) + + @staticmethod + def footprint_request( + identity: str, lane: PlanLane, warmth: Warmth, cost: float, saving: float | None = None + ) -> MeasurementRequest: + return MeasurementRequest( + identity=identity, + cell=lane.cell, + container_shape=lane.container_shape, + concurrency=lane.concurrency, + lane=lane.name, + measure=(Measurement.FOOTPRINT,), + warmth=warmth, + reason=RequestReason.UNMEASURED, + cost_bound=cost, + saving_bound=saving, + ) + def hourly(self, receipt: Receipt) -> float: return container_hourly_cost( self.prices[receipt.cell.machine_profile], receipt.container_shape, self.multiplier @@ -786,9 +824,10 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: lane for name, lane in sorted(self.current_lanes.items()) if (lazy if name == home else self.floor_paid(name, identity)) + and (identity, name) not in self.excluded and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells - and not self.hosts_generation((*self.current_pinned[name], *self.current_lazy[name])) + and not self.hosts_solitary((*self.current_pinned[name], *self.current_lazy[name])) and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -848,7 +887,7 @@ def current_gaps( missing.append((Measurement.RECEIPT, RequestReason.REFUSED if refused else RequestReason.UNMEASURED)) if entry.warmth in BOOT_WARMTHS and (identity, lane.cell, lane.container_shape, entry.warmth) not in self.boots: missing.append((Measurement.BOOT_EVIDENCE, RequestReason.UNMEASURED)) - return tuple( + requests = [ MeasurementRequest( identity=identity, cell=lane.cell, @@ -863,7 +902,18 @@ def current_gaps( co_residents=tuple(sorted(beside)) if measurement is Measurement.CO_RESIDENCY_RECEIPT else (), ) for measurement, reason in missing - ) + ] + if entry.warmth is Warmth.LAZY_RESIDENT: + pinned = self.current_pinned[lane.name] - {identity} + lazy = self.current_lazy[lane.name] | {identity} + if self.memory(lane, pinned=pinned, lazy=lazy).fits is None: + requests.extend( + self.footprint_request( + resident, lane, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT, cost + ) + for resident in self.memory_gaps(lane, (*pinned, *lazy)) + ) + return tuple(requests) def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice: beside = self.pinned_beside(identity) if beside is None else beside @@ -872,10 +922,10 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice options: list[_Option] = [] withdrawn: list[_Option] = [] boot_requests: dict[tuple[Cell, ContainerShape], MeasurementRequest] = {} - load_requests: dict[str, MeasurementRequest] = {} + load_requests: dict[tuple[str, str], MeasurementRequest] = {} receipts = self.receipts.get(identity, []) current_class = self.current_class(identity) - hosts = () if identity in self.generation else self.lazy_hosts(identity) + hosts = () if identity in self.solitary else self.lazy_hosts(identity) for order, receipt in receipts: hourly = self.hourly(receipt) rate = receipt.throughput.requests_per_second @@ -948,36 +998,49 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice continue lazy_cost = hourly * self.resident_hours(host, identity, requests, rate) load = self.boots.get((identity, host.cell, host.container_shape, Warmth.LAZY_RESIDENT)) - memory = self.memory( - host, - pinned=self.current_pinned[host.name] - {identity}, - lazy=self.current_lazy[host.name] | {identity}, - ) + pinned = self.current_pinned[host.name] - {identity} + lazy = self.current_lazy[host.name] | {identity} + memory = self.memory(host, pinned=pinned, lazy=lazy) if memory.fits is False: continue + lacking = () if memory.fits else self.memory_gaps(host, (*pinned, *lazy)) missing = ( *(() if load is not None and load.seconds is not None else (Measurement.BOOT_EVIDENCE,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), ) - if missing: - known = load_requests.get(host.name) - if not blocked and level.come_alive_p95_s != 0 and (known is None or lazy_cost < known.cost_bound): - load_requests[host.name] = MeasurementRequest( - identity=identity, - cell=host.cell, - container_shape=host.container_shape, - concurrency=host.concurrency, - lane=host.name, - measure=missing, - warmth=Warmth.LAZY_RESIDENT, - reason=( - RequestReason.FAILED - if load is not None and load.seconds is None - else RequestReason.UNMEASURED - ), - cost_bound=lazy_cost, - saving_bound=None, + if missing or lacking: + if blocked or level.come_alive_p95_s == 0: + continue + found = [ + self.footprint_request( + resident, host, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT, lazy_cost + ) + for resident in lacking + if resident != identity + ] + if missing: + found.append( + MeasurementRequest( + identity=identity, + cell=host.cell, + container_shape=host.container_shape, + concurrency=host.concurrency, + lane=host.name, + measure=missing, + warmth=Warmth.LAZY_RESIDENT, + reason=( + RequestReason.FAILED + if load is not None and load.seconds is None + else RequestReason.UNMEASURED + ), + cost_bound=lazy_cost, + saving_bound=None, + ) ) + for request in found: + known = load_requests.get((host.name, request.identity)) + if known is None or lazy_cost < known.cost_bound: + load_requests[(host.name, request.identity)] = request continue if load is None or load.seconds is None: continue @@ -1016,22 +1079,13 @@ def choose(self, identity: str, beside: frozenset[str] | None = None) -> _Choice held = cost unpriced = cost is None gaps = self.current_gaps(identity, entry, lane, cost, beside) - gapped = { - (request.cell, request.container_shape, request.warmth) - for request in gaps - if Measurement.BOOT_EVIDENCE in request.measure - } return _Choice( identity=identity, level=level, requests=requests, options=tuple(options), chosen=chosen, - boot_requests=tuple( - request - for request in (*boot_requests.values(), *load_requests.values()) - if (request.cell, request.container_shape, request.warmth) not in gapped - ), + boot_requests=_ungapped(identity, (*boot_requests.values(), *load_requests.values()), gaps), has_valid_receipt=bool(receipts), held=held, gaps=gaps, @@ -1523,22 +1577,13 @@ def stay(self, choice: _Choice) -> _Choice | None: return None cost = self.current_cost(identity, entry, lane) gaps = self.current_gaps(identity, entry, lane, cost, choice.beside) - gapped = { - (request.cell, request.container_shape, request.warmth) - for request in gaps - if Measurement.BOOT_EVIDENCE in request.measure - } return replace( choice, chosen=None, held=cost, unpriced=cost is None, gaps=gaps, - boot_requests=tuple( - request - for request in choice.boot_requests - if (request.cell, request.container_shape, request.warmth) not in gapped - ), + boot_requests=_ungapped(identity, choice.boot_requests, gaps), ) @staticmethod @@ -1589,13 +1634,14 @@ def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, f (lane, frozenset(pinned[lane.name]), frozenset(lazy[lane.name])) for lane in layout.plan.lanes if lane.name != layout.names[identity] + and (identity, lane.name) not in self.excluded and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and lane.container_shape.gpu_count == 1 and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and pinned[lane.name] - and not self.hosts_generation((*pinned[lane.name], *lazy[lane.name])) + and not self.hosts_solitary((*pinned[lane.name], *lazy[lane.name])) and self.slowest_receipt(identity, lane) is not None ] return sorted( @@ -1613,7 +1659,7 @@ def packings( target: tuple[PlanLane, frozenset[str], frozenset[str]], layout: _Layout, choices: Mapping[str, _Choice], - ) -> list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]]: + ) -> list[_Packing]: lane, pinned, lazy = target level = self.levels[identity] order, receipt = min( @@ -1625,7 +1671,7 @@ def packings( offered = choices[identity] hosts = {option.host for option in offered.options if option.warmth is Warmth.LAZY_RESIDENT} hosts.update(request.lane for request in offered.boot_requests if request.warmth is Warmth.LAZY_RESIDENT) - packings: list[tuple[dict[str, _Option], tuple[Measurement, ...], tuple[str, ...]]] = [] + packings: list[_Packing] = [] current_class = self.current_class(identity) blocked = (*self.warm_blockers(level, receipt, current_class), *self.reliability_blockers(level, lane.cells)) load = self.boots.get((identity, lane.cell, lane.container_shape, Warmth.LAZY_RESIDENT)) @@ -1633,14 +1679,16 @@ def packings( level.come_alive_p95_s != 0 and not blocked and lane.name not in hosts + and any(self.levels[resident].come_alive_p95_s == 0 for resident in pinned) and (load is None or load.seconds is not None) and (load is None or load.seconds is None or _come_alive_blocker(level, load.seconds) is None) ): memory = self.memory(lane, pinned=pinned, lazy=lazy | {identity}) if memory.fits is not False: + lacking = () if memory.fits else self.memory_gaps(lane, (*pinned, *lazy, identity)) missing = ( *(() if load is not None else (Measurement.BOOT_EVIDENCE,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), ) option = _Option( receipt=receipt, @@ -1655,7 +1703,12 @@ def packings( host=host, host_owner=owner, ) - packings.append(({identity: option}, missing, ())) + others = tuple( + (resident, Warmth.PINNED if resident in pinned else Warmth.LAZY_RESIDENT) + for resident in lacking + if resident != identity + ) + packings.append(({identity: option}, missing, (), others)) members = pinned | {identity} if all(choices[member].chosen is not None for member in pinned): together = {member: self.co_receipt(member, lane, members - {member}) for member in members} @@ -1669,9 +1722,15 @@ def packings( ): memory = self.memory(lane, pinned=members, lazy=lazy) if memory.fits is not False: + lacking = () if memory.fits else self.memory_gaps(lane, (*members, *lazy)) missing = ( *(() if len(measured) == len(members) else (Measurement.CO_RESIDENCY_RECEIPT,)), - *(() if memory.fits else (Measurement.FOOTPRINT,)), + *((Measurement.FOOTPRINT,) if identity in lacking else ()), + ) + others = tuple( + (resident, Warmth.PINNED if resident in members else Warmth.LAZY_RESIDENT) + for resident in lacking + if resident != identity ) own_order, own = measured.get(identity, (order, receipt)) updates = { @@ -1688,7 +1747,7 @@ def packings( host_owner=owner, ) } - if not missing: + if not missing and not others: for member in sorted(pinned): chosen = choices[member].chosen member_order, member_receipt = measured[member] @@ -1701,7 +1760,7 @@ def packings( cost=self.hourly(member_receipt) * self.window.warm_container_hours(choices[member].requests, rate), ) - packings.append((updates, missing, tuple(sorted(pinned)))) + packings.append((updates, missing, tuple(sorted(pinned)), others)) return packings def pack(self, choices: dict[str, _Choice]) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: @@ -1721,19 +1780,18 @@ def floor_cost(identity: str) -> float: identity for identity, choice in choices.items() if (chosen := choice.chosen) is not None - and chosen.host is None + and chosen.host in (None, layout.names[identity]) and chosen.host_owner is None - and chosen.warmth is not Warmth.LAZY_RESIDENT - and identity not in self.generation + and identity not in self.solitary and hosted[layout.names[identity]] == 1 ] - requests: dict[tuple[str, str, Warmth], MeasurementRequest] = {} + requests: dict[tuple[str, str | None, Warmth, tuple[Measurement, ...]], MeasurementRequest] = {} for identity in sorted(candidates, key=lambda identity: (-floor_cost(identity), identity)): if hosted.get(layout.names.get(identity, ""), 0) != 1: continue moved = False for target in self.pack_targets(identity, layout): - for updates, missing, beside in self.packings(identity, target, layout, choices): + for updates, missing, beside, lacking in self.packings(identity, target, layout, choices): members = frozenset(beside) | {identity} trial = { **choices, @@ -1750,23 +1808,32 @@ def floor_cost(identity: str) -> float: if round(saving, 2) <= 0: continue option = updates[identity] - if missing: - key = (identity, target[0].name, option.warmth) - known = requests.get(key) - if known is None or saving > (known.saving_bound or 0.0): - requests[key] = MeasurementRequest( - identity=identity, - cell=target[0].cell, - container_shape=target[0].container_shape, - concurrency=target[0].concurrency, - lane=target[0].name, - measure=missing, - warmth=option.warmth, - reason=RequestReason.UNMEASURED, - cost_bound=option.cost, - saving_bound=saving, - co_residents=beside, + if missing or lacking: + found = [ + self.footprint_request(resident, target[0], warmth, option.cost, saving) + for resident, warmth in lacking + ] + if missing: + found.append( + MeasurementRequest( + identity=identity, + cell=target[0].cell, + container_shape=target[0].container_shape, + concurrency=target[0].concurrency, + lane=target[0].name, + measure=missing, + warmth=option.warmth, + reason=RequestReason.UNMEASURED, + cost_bound=option.cost, + saving_bound=saving, + co_residents=beside, + ) ) + for request in found: + key = (request.identity, request.lane, request.warmth, request.measure) + known = requests.get(key) + if known is None or saving > (known.saving_bound or 0.0): + requests[key] = request continue choices, layout, total = trial, trial_layout, trial_total hosted = Counter(entry.lane for entry in layout.plan.identities) @@ -1801,13 +1868,56 @@ def settle(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: choices = {**choices, **{identity: self.choose(identity, beside) for identity, beside in stale.items()}} return choices + def overflowing(self, choices: Mapping[str, _Choice]) -> list[tuple[str, str]]: + layout = self.assemble(choices) + before: defaultdict[str, set[str]] = defaultdict(set) + if self.current_plan is not None: + for entry in self.current_plan.identities: + before[entry.lane].add(entry.identity) + residents: defaultdict[str, dict[str, Warmth]] = defaultdict(dict) + for entry in layout.plan.identities: + residents[entry.lane][entry.identity] = entry.warmth + rejected: list[tuple[str, str]] = [] + for lane in layout.plan.lanes: + hosted = residents[lane.name] + joiners = sorted( + ( + identity + for identity in hosted + if identity not in before[lane.name] + and (chosen := choices[identity].chosen) is not None + and (chosen.host is not None or chosen.host_owner is not None) + ), + key=lambda identity: (-self.footprints.get((identity, lane.cell), 0), identity), + ) + if not joiners or self.lane_memory(lane, hosted).fits: + continue + admitted = {identity: warmth for identity, warmth in hosted.items() if identity not in joiners} + for identity in joiners: + trial = {**admitted, identity: hosted[identity]} + if self.lane_memory(lane, trial).fits: + admitted = trial + else: + rejected.append((identity, lane.name)) + return rejected + + def fit(self, choices: dict[str, _Choice]) -> dict[str, _Choice]: + for _ in range(len(self.levels) * (len(self.current_lanes) + len(self.levels)) + 1): + choices = self.settle(choices) + rejected = self.overflowing(choices) + if not rejected: + return choices + self.excluded.update(rejected) + choices = {**choices, **{identity: self.choose(identity) for identity, _ in rejected}} + return self.settle(choices) + def decide(self) -> tuple[dict[str, _Choice], list[MeasurementRequest]]: never_cold = [identity for identity in sorted(self.levels) if self.levels[identity].come_alive_p95_s == 0] self.anchors = self.staying_anchors(self.choose(identity) for identity in never_cold) originals = {identity: self.choose(identity) for identity in sorted(self.levels)} - decided = self.stay_beside(self.settle(dict(originals)), originals) + decided = self.fit(self.stay_beside(self.settle(dict(originals)), originals)) decided, requests = self.pack(decided) - return self.settle(decided), requests + return self.fit(decided), requests def run(self, *, price_savings: bool = True) -> PlannerResult: decided, packing_requests = self.decide() @@ -1843,7 +1953,7 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: identities=tuple(reports), lanes=lane_reports, shared_chart_lanes=_shared_chart_lanes(new_plan), - measurement_requests=tuple(sorted(requests, key=_request_rank)), + measurement_requests=_distinct(requests), come_alive_levers=tuple(sorted(levers, key=_lever_rank)), queue_wait_unattained=tuple( identity for identity in sorted(self.levels) if self.levels[identity].queue_wait_p99_ms is not None @@ -1858,6 +1968,7 @@ def run(self, *, price_savings: bool = True) -> PlannerResult: ), capacity=self.capacity_use(new_plan), scarcity=tuple(sorted(scarcity, key=_scarcity_rank)), + gpu_memory_undeclared=tuple(sorted(self.gpu_memory_undeclared)), ) return PlannerResult(plan=new_plan, report=report) @@ -1951,6 +2062,38 @@ def _pricing( return tuple(reports), costs +def _ungapped( + identity: str, requests: Iterable[MeasurementRequest], gaps: Iterable[MeasurementRequest] +) -> tuple[MeasurementRequest, ...]: + gapped = { + (request.cell, request.container_shape, request.warmth) + for request in gaps + if request.identity == identity and Measurement.BOOT_EVIDENCE in request.measure + } + return tuple( + request + for request in requests + if request.identity != identity or (request.cell, request.container_shape, request.warmth) not in gapped + ) + + +def _distinct(requests: Iterable[MeasurementRequest]) -> tuple[MeasurementRequest, ...]: + kept: dict[tuple[object, ...], MeasurementRequest] = {} + for request in sorted(requests, key=_request_rank): + key = ( + request.identity, + request.cell, + request.container_shape, + request.concurrency, + request.lane, + request.measure, + request.warmth, + request.co_residents, + ) + kept.setdefault(key, request) + return tuple(kept.values()) + + def _scarcity_reason(option: _Option, bound: int | None) -> ScarcityReason | None: fields = _fields(option.blockers) if fields == [PLACEMENT_FAILURE_FIELD]: diff --git a/packages/sie_placement/src/sie_placement/report.py b/packages/sie_placement/src/sie_placement/report.py index 430fdcda0..338d50f97 100644 --- a/packages/sie_placement/src/sie_placement/report.py +++ b/packages/sie_placement/src/sie_placement/report.py @@ -428,6 +428,7 @@ class Report: warm_bound_undeclared: tuple[str, ...] = () capacity: tuple[CapacityUse, ...] = () scarcity: tuple[Scarcity, ...] = () + gpu_memory_undeclared: tuple[str, ...] = () @property def monthly_cost(self) -> float: @@ -472,6 +473,7 @@ def to_dict(self) -> dict[str, object]: "scarcity": [entry.to_dict() for entry in self.scarcity], "queueWaitUnattained": list(self.queue_wait_unattained), "warmBoundUndeclared": list(self.warm_bound_undeclared), + "gpuMemoryUndeclared": list(self.gpu_memory_undeclared), "refusals": [refusal.to_dict() for refusal in self.refusals], "changes": { "identities": [change.to_dict() for change in self.identity_changes], diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 91a4f0861..226647ae1 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -482,13 +482,33 @@ "concurrency": 16, "lane": "default-l4-default", "measure": [ - "boot-evidence", - "footprint" + "boot-evidence" ], "warmth": "lazy-resident", "reason": "unmeasured", "costBound": 0.0, "savingBound": 47.45 + }, + { + "identity": "intfloat/e5-base-v2", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "lane": "default-l4-default", + "measure": [ + "footprint" + ], + "warmth": "pinned", + "reason": "unmeasured", + "costBound": 0.0, + "savingBound": 47.45 } ], "comeAliveLevers": [ @@ -540,6 +560,7 @@ "intfloat/e5-base-v2" ], "warmBoundUndeclared": [], + "gpuMemoryUndeclared": [], "refusals": [ { "record": "boot-evidence", diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index 2449df044..ae04e6feb 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -276,15 +276,85 @@ def test_a_packing_without_footprints_is_a_measurement_request_with_its_saving_b assert requests[1].to_dict()["measure"] == ["footprint"] -def test_a_packing_without_a_load_asks_for_the_load_and_the_footprints_it_lacks(): +def test_a_packing_asks_the_joiner_for_its_load_and_each_resident_for_the_memory_evidence_it_lacks(): receipts = [receipt(HOST), receipt(GUEST, footprint=4000, working=1500)] result = run(levels(**{HOST: 0, GUEST: 60}), receipts) - lazy = [ - request + on_host = [request for request in result.report.measurement_requests if request.lane == "org-host-l4"] + assert [(request.identity, request.warmth, request.measure) for request in on_host] == [ + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.BOOT_EVIDENCE,)), + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT,)), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,)), + ] + assert all(request.saving_bound == pytest.approx(MONTH) for request in on_host) + + +def test_a_host_resident_without_memory_evidence_is_asked_for_it_under_its_own_identity(): + current = plan_of([lane("host"), lane("own")], {HOST: ("host", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + receipts = [receipt(HOST), receipt(GUEST, footprint=1000, working=500)] + result = run(levels(**{HOST: 0, GUEST: 60}), receipts, [load(GUEST)], current=current) + assert placements(result)[GUEST] == ("own", Warmth.PINNED) + on_host = [request for request in result.report.measurement_requests if request.lane == "host"] + assert [(request.identity, request.warmth, request.measure) for request in on_host] == [ + (GUEST, Warmth.PINNED, (Measurement.CO_RESIDENCY_RECEIPT,)), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,)), + ] + assert on_host[1].saving_bound == pytest.approx(MONTH) + + +def test_a_resident_several_joiners_need_measured_is_asked_once(): + current = plan_of( + [lane("host"), lane("own-a"), lane("own-b")], + {HOST: ("host", Warmth.PINNED), "org/a": ("own-a", Warmth.PINNED), "org/b": ("own-b", Warmth.PINNED)}, + ) + receipts = [ + receipt(HOST), + receipt("org/a", footprint=1000, working=500), + receipt("org/b", footprint=1000, working=500), + ] + result = run( + levels(**{HOST: 0, "org/a": 60, "org/b": 60}), receipts, [load("org/a"), load("org/b")], current=current + ) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + "org/a": ("own-a", Warmth.PINNED), + "org/b": ("own-b", Warmth.PINNED), + } + asked = [request for request in result.report.measurement_requests if request.lane == "host"] + assert [(request.identity, request.measure, request.warmth) for request in asked] == [ + ("org/a", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED), + ("org/b", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED), + (HOST, (Measurement.FOOTPRINT,), Warmth.PINNED), + ] + assert asked[2].saving_bound == pytest.approx(MONTH) + + +@pytest.mark.parametrize("price_list", [prices(), prices(cards=False)], ids=["no-footprints", "no-card-memory"]) +def test_a_current_lazy_resident_whose_lane_lacks_memory_evidence_is_kept_and_says_what_is_missing(price_list): + current = plan_of([lane("host")], {HOST: ("host", Warmth.PINNED), GUEST: ("host", Warmth.LAZY_RESIDENT)}) + result = run( + levels(**{HOST: 0, GUEST: 60}), + [receipt(HOST), receipt(GUEST)], + [load(GUEST)], + current=current, + price_list=price_list, + ) + assert result.plan == current + report = result.report.identity(GUEST) + assert (report.status, report.unplaced_reason) == (IdentityStatus.KEPT, UnplacedReason.CURRENT_UNMEASURED) + requests = [ + (request.identity, request.warmth, request.measure, request.saving_bound) for request in result.report.measurement_requests - if request.lane == "org-host-l4" and request.warmth is Warmth.LAZY_RESIDENT ] - assert [request.measure for request in lazy] == [(Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT)] + if price_list.machine_profiles[0].gpu_memory_mib is None: + assert requests == [] + assert result.report.gpu_memory_undeclared == ("l4",) + assert result.report.to_dict()["gpuMemoryUndeclared"] == ["l4"] + return + assert requests == [ + (GUEST, Warmth.LAZY_RESIDENT, (Measurement.FOOTPRINT,), None), + (HOST, Warmth.PINNED, (Measurement.FOOTPRINT,), None), + ] + assert result.report.gpu_memory_undeclared == () @pytest.mark.parametrize( @@ -444,6 +514,32 @@ def test_packing_fills_the_lane_with_the_most_footprint_in_use_first(): assert ("org/h2", "org-h1-l4", Warmth.PINNED) in requests +@pytest.mark.parametrize(("bound", "joined"), [(0, True), (60, False)]) +def test_packing_makes_a_lazy_resident_only_beside_a_never_cold_pinned_identity(bound, joined): + receipts = [receipt("org/warm", footprint=9000, working=1000), receipt(GUEST, footprint=2000, working=1000)] + result = run(levels(**{"org/warm": bound, GUEST: 60}), receipts, [load(GUEST)]) + if joined: + assert placements(result)[GUEST] == ("org-warm-l4", Warmth.LAZY_RESIDENT) + return + assert placements(result) == { + "org/warm": ("org-warm-l4", Warmth.PINNED), + GUEST: ("org-guest-l4", Warmth.PINNED), + } + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + + +def test_a_lazy_resident_alone_on_its_own_lane_is_packed_beside_a_never_cold_identity_in_the_same_run(): + current = plan_of([lane("own")], {GUEST: ("own", Warmth.LAZY_RESIDENT)}) + receipts = [receipt(HOST, footprint=5000, working=500), receipt(GUEST, footprint=5000, working=500)] + service_levels = levels(**{HOST: 0, GUEST: 600}) + result = run(service_levels, receipts, [load(GUEST)], current=current) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert result.report.monthly_cost == pytest.approx(MONTH) + again = run(service_levels, receipts, [load(GUEST)], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + def test_packing_tries_lanes_a_never_cold_identity_pins_before_the_others(): receipts = [ receipt("org/a-warm", footprint=9000, working=1000), @@ -494,6 +590,67 @@ def test_the_lazy_resident_selection_on_a_current_lane_requires_the_memory_check assert requests == ([measure] if measure else []) +def joining(residents): + current = plan_of( + [lane("host"), *(lane(f"own-{identity[-1]}") for identity in residents)], + {HOST: ("host", Warmth.PINNED), **{identity: (f"own-{identity[-1]}", Warmth.PINNED) for identity in residents}}, + ) + receipts = [receipt(HOST, footprint=6000, working=1000)] + receipts.extend(receipt(identity, footprint=size, working=work) for identity, (size, work) in residents.items()) + boots = [load(identity) for identity in residents] + return levels(**{HOST: 0, **dict.fromkeys(residents, 60)}), receipts, boots, current + + +@pytest.mark.parametrize( + ("residents", "joined", "memory"), + [ + ({"org/a": (8000, 1000), "org/b": (1000, 8000)}, {"org/a"}, 6000 + 8000 + 1000), + ({"org/a": (7000, 1000), "org/b": (2000, 6000), "org/c": (3000, 1000)}, {"org/a", "org/c"}, 6000 + 7000 + 1000), + ], + ids=["two", "three"], +) +def test_lazy_residents_that_each_fit_a_lane_alone_join_it_only_as_far_as_they_fit_together(residents, joined, memory): + for identity, sizes in residents.items(): + alone = run(*joining({identity: sizes})[:3], current=joining({identity: sizes})[3]) + assert placements(alone)[identity] == ("host", Warmth.LAZY_RESIDENT) + service_levels, receipts, boots, current = joining(residents) + result = run(service_levels, receipts, boots, current=current) + assert placements(result) == { + HOST: ("host", Warmth.PINNED), + **{ + identity: ("host", Warmth.LAZY_RESIDENT) if identity in joined else (f"own-{identity[-1]}", Warmth.PINNED) + for identity in residents + }, + } + host = {lane_report.name: lane_report for lane_report in result.report.lanes}["host"] + assert (host.memory_mib, host.memory_limit_mib) == (memory, pytest.approx(LIMIT)) + again = run(service_levels, receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +@pytest.mark.parametrize("unshared", [GUEST, HOST]) +def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(unshared): + service_levels = levels(**{HOST: 0, GUEST: 60}) + result = run(service_levels, host_and_guest(), [load(GUEST)], unshared=frozenset({unshared})) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-guest-l4", Warmth.PINNED)} + assert [request for request in result.report.measurement_requests if request.lane is not None] == [] + assert result.report.parameters["unshared"] == [unshared] + current = plan_of([lane("host"), lane("own")], {HOST: ("host", Warmth.PINNED), GUEST: ("own", Warmth.PINNED)}) + kept = run(service_levels, host_and_guest(), [load(GUEST)], current=current, unshared=frozenset({unshared})) + assert kept.plan == current + assert all(request.lane != "host" for request in kept.report.measurement_requests) + shared = run(service_levels, host_and_guest(), [load(GUEST)], current=current) + assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) + + +def test_an_unshared_identity_keeps_a_current_shared_lane_it_already_holds(): + current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current, unshared=frozenset({"org/a"})) + assert result.plan == current + assert result.report.unchanged + + def test_an_identity_whose_neighbours_departure_strips_its_co_residency_evidence_is_decided_again_in_the_run(): current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) receipts = [ @@ -577,9 +734,23 @@ def test_identities_that_leave_a_lane_together_return_when_sharing_it_costs_less IDENTITIES = ("org/m0", "org/m1", "org/m2", "org/m3", "org/m4") -@pytest.mark.parametrize("seed", range(150)) -def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): - rng = random.Random(seed) # noqa: S311 -- reproducible test data +def scattered(rng, identities, generation) -> Plan | None: + names = [f"current-{index}" for index in range(rng.randint(1, 3))] + residents = { + identity: (rng.choice(names), rng.choice([Warmth.PINNED, Warmth.PINNED, Warmth.LAZY_RESIDENT])) + for identity in identities + if identity not in generation and rng.random() < 0.8 + } + residents.update( + (identity, (f"alone-{identity[-1]}", Warmth.PINNED)) for identity in sorted(generation) if rng.random() < 0.8 + ) + used = sorted({name for name, _ in residents.values()}) + if not used: + return None + return plan_of([lane(name, rng.choice([L4, L4, A100])) for name in used], residents) + + +def random_estate(rng): identities = list(IDENTITIES[: rng.randint(2, 5)]) generation = {identity for identity in identities if rng.random() < 0.15} entries = [] @@ -594,8 +765,8 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): ) ) service_levels = ServiceLevels(identities=tuple(entries)) - footprints = {identity: rng.choice([None, 2000, 5000, 9000]) for identity in identities} - working = {identity: rng.choice([None, 500, 2500]) for identity in identities} + footprints = {identity: rng.choice([None, 2000, 5000, 9000, 12000]) for identity in identities} + working = {identity: rng.choice([None, 500, 2500, 8000]) for identity in identities} receipts = [] for identity in identities: for cell in (L4, A100): @@ -633,31 +804,39 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): identity: {hour: rng.choice([1, 500]) for hour in rng.sample(range(24), rng.randint(0, 3))} for identity in identities } + unshared = frozenset(identity for identity in plain if rng.random() < 0.15) + draw = rng.random() current = None - if rng.random() < 0.5: - first = run(service_levels, receipts, boots, hourly=hourly) - current = first.plan - result = run(service_levels, receipts, boots, hourly=hourly, current=current) + if draw < 0.35: + current = run(service_levels, receipts, boots, hourly=hourly, unshared=unshared).plan + elif draw < 0.7: + current = scattered(rng, identities, generation) + return service_levels, receipts, boots, hourly, current, unshared, generation, not 0.35 <= draw < 0.7 + + +@pytest.mark.parametrize("seed", range(200)) +def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + service_levels, receipts, boots, hourly, current, unshared, generation, settled = random_estate(rng) + result = run(service_levels, receipts, boots, hourly=hourly, current=current, unshared=unshared) assert Plan.from_dict(result.plan.to_dict()) == result.plan levels_by_identity = {entry.identity: entry.level for entry in service_levels.identities} residents: dict[str, list[PlanIdentity]] = {} for entry in result.plan.identities: residents.setdefault(entry.lane, []).append(entry) - before = {} if current is None else {planned.name: planned for planned in current.lanes} + before: dict[str, set[str]] = {} + for entry in () if current is None else current.identities: + before.setdefault(entry.lane, set()).add(entry.identity) reports = {lane_report.name: lane_report for lane_report in result.report.lanes} for planned in result.plan.lanes: hosted = residents[planned.name] - if any(entry.identity in generation for entry in hosted): - assert len(hosted) == 1 if len(hosted) < 2: continue pinned = {entry.identity for entry in hosted if entry.warmth is Warmth.PINNED} - formed = planned.name not in before or { - entry.identity for entry in current.identities if entry.lane == planned.name - } != {entry.identity for entry in hosted} - if formed: + if {entry.identity for entry in hosted} - before.get(planned.name, set()): assert reports[planned.name].memory_mib is not None assert reports[planned.name].memory_mib <= reports[planned.name].memory_limit_mib + assert not any(entry.identity in generation | unshared for entry in hosted) for entry in hosted: report = result.report.identity(entry.identity) if report.chosen is None: @@ -672,13 +851,14 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): assert level.come_alive_p95_s is None or report.chosen.come_alive_s <= level.come_alive_p95_s for request in result.report.measurement_requests: assert request.saving_bound is None or request.saving_bound > 0 - again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan) - assert again.plan == result.plan - assert again.report.unchanged - assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) + if settled: + again = run(service_levels, receipts, boots, hourly=hourly, current=result.plan, unshared=unshared) + assert again.plan == result.plan + assert again.report.unchanged + assert again.report.monthly_cost == pytest.approx(result.report.monthly_cost) shuffled = list(receipts) rng.shuffle(shuffled) - reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current) + reordered = run(service_levels, shuffled, list(reversed(boots)), hourly=hourly, current=current, unshared=unshared) assert rendered(reordered) == rendered(result) for entry in service_levels.identities: bound = entry.level.come_alive_p95_s @@ -690,5 +870,44 @@ def test_random_estates_pack_only_what_fits_and_holds_every_service_level(seed): for other in service_levels.identities ) ) - loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan) + loosened = run(looser, receipts, boots, hourly=hourly, current=result.plan, unshared=unshared) assert loosened.report.monthly_cost <= result.report.monthly_cost + 0.01 + + +def need(host, guests, sizes): + return ( + host[0] + + max((sizes[guest][0] for guest in guests), default=0) + + max((host[1], *(sizes[guest][1] for guest in guests))) + ) + + +@pytest.mark.parametrize("seed", range(150)) +def test_random_guests_join_a_host_lane_only_as_far_as_they_fit_together(seed): + rng = random.Random(seed) # noqa: S311 -- reproducible test data + host = (rng.choice([4000, 6000, 8000]), rng.choice([500, 1000, 2000])) + sizes = { + f"org/g{index}": (rng.choice([1000, 3000, 6000, 9000]), rng.choice([500, 2000, 6000, 9000])) + for index in range(rng.randint(2, 4)) + } + current = plan_of( + [lane("host"), *(lane(f"own-{guest[-1]}") for guest in sizes)], + {HOST: ("host", Warmth.PINNED), **{guest: (f"own-{guest[-1]}", Warmth.PINNED) for guest in sizes}}, + ) + receipts = [receipt(HOST, footprint=host[0], working=host[1])] + receipts.extend(receipt(guest, footprint=size, working=work) for guest, (size, work) in sizes.items()) + boots = [load(guest) for guest in sizes] + service_levels = levels(**{HOST: 0, **dict.fromkeys(sizes, 60)}) + result = run(service_levels, receipts, boots, current=current) + joined = {guest for guest, (name, _) in placements(result).items() if name == "host" and guest != HOST} + assert all(placements(result)[guest] == ("host", Warmth.LAZY_RESIDENT) for guest in joined) + assert need(host, joined, sizes) <= LIMIT + for guest in sizes.keys() - joined: + assert placements(result)[guest] == (f"own-{guest[-1]}", Warmth.PINNED) + assert need(host, joined | {guest}, sizes) > LIMIT + if joined: + (report,) = [lane_report for lane_report in result.report.lanes if lane_report.name == "host"] + assert report.memory_mib == need(host, joined, sizes) + again = run(service_levels, receipts, boots, current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index c4bf3886c..e0d78d211 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1199,8 +1199,16 @@ def test_a_lazy_resident_without_evidence_keeps_its_place_and_asks_for_it(): ] assert requests == [ ((Measurement.BOOT_EVIDENCE,), Warmth.LAZY_RESIDENT, "shared", 8, None), + ((Measurement.FOOTPRINT,), Warmth.LAZY_RESIDENT, "shared", 8, None), ((Measurement.RECEIPT,), Warmth.LAZY_RESIDENT, "shared", 8, None), ] + (anchor,) = [request for request in result.report.measurement_requests if request.identity == ANCHOR] + assert (anchor.measure, anchor.warmth, anchor.lane, anchor.saving_bound) == ( + (Measurement.FOOTPRINT,), + Warmth.PINNED, + "shared", + None, + ) def test_a_measured_load_onto_a_lane_a_never_cold_neighbour_pins_makes_a_lazy_resident(): @@ -1335,17 +1343,22 @@ def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): assert all(request.lane != "shared" for request in result.report.measurement_requests) -def test_a_current_lane_hosts_lazy_residents_beside_a_never_cold_neighbour_and_a_plan_lane_beside_any_pinned_one(): +def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_current_lane_and_on_a_packed_one(): current = plan_of( [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} ) result = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) report = result.report.identity(IDENTITY) - assert (report.lane, report.chosen.warmth, report.chosen.lane) == ("shared", Warmth.LAZY_RESIDENT, "shared") - assert result.report.monthly_cost == pytest.approx(MONTH) + assert (report.lane, report.chosen.warmth) == ("own", Warmth.PINNED) + assert result.report.monthly_cost == pytest.approx(2 * MONTH) assert all(lever.warmth is not Warmth.LAZY_RESIDENT for lever in result.report.come_alive_levers) - again = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=result.plan) - assert again.report.unchanged + assert all(request.warmth is not Warmth.LAZY_RESIDENT for request in result.report.measurement_requests) + fresh = run(resident_levels(anchor_bound=30), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY) + assert {entry.identity: entry.warmth for entry in fresh.plan.identities} == { + ANCHOR: Warmth.PINNED, + IDENTITY: Warmth.PINNED, + } + assert len(fresh.plan.lanes) == 2 @pytest.mark.parametrize( @@ -1546,6 +1559,9 @@ def test_parameters_echo_into_the_report(): "bundles": {"org/a": "default", "org/b": "sglang"}, } assert list(echoed["bundles"]) == ["org/a", "org/b"] + assert parameters(unshared=frozenset({"org/b", "org/a"})).to_dict()["unshared"] == ["org/a", "org/b"] + with pytest.raises(SchemaError, match="unshared"): + parameters(unshared=frozenset({"org/a b"})) def test_the_placement_must_be_in_the_price_list(): @@ -1646,11 +1662,18 @@ def test_the_estate_report_states_its_findings(): ("intfloat/e5-base-v2", (Measurement.CO_RESIDENCY_RECEIPT,), Warmth.PINNED, "default-l4-default", None), ( "BAAI/bge-m3:sparse", - (Measurement.BOOT_EVIDENCE, Measurement.FOOTPRINT), + (Measurement.BOOT_EVIDENCE,), Warmth.LAZY_RESIDENT, "default-l4-default", pytest.approx(sparse.chosen.monthly_cost), ), + ( + "intfloat/e5-base-v2", + (Measurement.FOOTPRINT,), + Warmth.PINNED, + "default-l4-default", + pytest.approx(sparse.chosen.monthly_cost), + ), ] assert dense.chosen.co_residents == ("intfloat/e5-base-v2",) assert dense.chosen.receipt.uri == "receipts/baai-bge-m3/l4-c16-16384-beside-e5-base-v2.json" From e8d0f5e6edcef77148036047e54e290118ef6592 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:03:24 +0100 Subject: [PATCH 03/10] test(placement): hold shared lanes to the ceilings at maxContainers - A shared lane holds its container ceiling once against the total, its GPU type and the shared quota, whichever residents it carries, so a packing fits where the separate lanes would be refused. - A shared lane over a ceiling is refused with its maxContainers intact, and the refusal names it. --- packages/sie_placement/tests/test_packing.py | 48 ++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index ae04e6feb..b3287c54f 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -5,11 +5,12 @@ from datetime import UTC, datetime, timedelta import pytest -from sie_placement.planner import PlannerParameters, plan +from sie_placement.planner import CapacityExceededError, PlannerParameters, plan from sie_placement.report import IdentityStatus, Measurement, RequestReason, UnplacedReason, render from sie_placement.schema import ( BootEvidence, BootOutcome, + Capacity, Cell, ContainerShape, Demand, @@ -29,8 +30,10 @@ RequestShape, ServiceLevel, ServiceLevels, + SharedQuota, SourceRef, Throughput, + TypeCeiling, Warmth, ) @@ -169,11 +172,15 @@ def demand(hourly=None) -> Demand: ) -def run(service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, **overrides): +def run( + service_levels, receipts=(), boots=(), *, current=None, hourly=None, price_list=None, capacity=None, **overrides +): parameters = PlannerParameters( **({"placement": "test", "max_containers": 4, "fallback_floor": 2, "cooldown_s": 900} | overrides) ) - return plan(service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters) + return plan( + service_levels, receipts, boots, price_list or prices(), demand(hourly), current, parameters, capacity=capacity + ) def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: @@ -644,6 +651,41 @@ def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(uns assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) +def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): + capacity = Capacity( + total=4, + types=(TypeCeiling(gpu_product=L4.gpu_product, accelerators=4),), + shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + ) + service_levels = levels(**{HOST: 0, GUEST: 60}) + result = run(service_levels, host_and_guest(), [load(GUEST)], capacity=capacity) + assert placements(result) == {HOST: ("org-host-l4", Warmth.PINNED), GUEST: ("org-host-l4", Warmth.LAZY_RESIDENT)} + assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ + (1, 4, ("org-host-l4",)), + (1, 4, ("org-host-l4",)), + (3, 6, ("org-host-l4",)), + ] + with pytest.raises(CapacityExceededError) as caught: + run(service_levels, host_and_guest(guest_footprint=8000), [load(GUEST)], capacity=capacity) + assert [use.describe() for use in caught.value.exceeded] == [ + "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, above the total ceiling of 4", + "the container ceilings of org-guest-l4, org-host-l4 hold 8 NVIDIA-L4 accelerators, above the type ceiling of 4", + "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, " + "which with 2 held elsewhere exceed the shared quota of 6", + ] + + +def test_a_shared_lane_over_a_ceiling_is_refused_with_its_container_ceiling_intact(): + with pytest.raises(CapacityExceededError) as caught: + run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)], capacity=Capacity(total=3)) + assert ( + str(caught.value) == "the container ceilings of org-host-l4 hold 4 accelerators, above the total ceiling of 3" + ) + (refused,) = caught.value.result.plan.lanes + assert (refused.name, refused.max_containers) == ("org-host-l4", 4) + assert caught.value.result.plan == run(levels(**{HOST: 0, GUEST: 60}), host_and_guest(), [load(GUEST)]).plan + + def test_an_unshared_identity_keeps_a_current_shared_lane_it_already_holds(): current = plan_of([lane("shared")], {"org/a": ("shared", Warmth.PINNED), "org/b": ("shared", Warmth.PINNED)}) result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), current=current, unshared=frozenset({"org/a"})) From 85313d89c1e0c95d23eca3b46cde0b411544d482 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 5 Nov 2026 19:30:26 +0100 Subject: [PATCH 04/10] fix(placement): put each pinning lane on a named queue pool of its own The sie-cluster chart carries a lane's pinned set on a static queue pool, and the default queue pool carries no pins, so a plan with more than one pinned lane on one queue pool could not be emitted as Helm values. The planner now gives every lane that pins an identity a named queue pool derived from the lane name, unique within the plan, keeps a named pool the current plan already gives the lane, and leaves lazy-only and cold lanes on the parameters' queue pool. A lane's queue pool no longer decides whether a current placement, a lazy host or a packing target is reusable, since the pool follows from the lane. --- packages/sie_placement/README.md | 21 ++- .../src/sie_placement/planner.py | 39 ++++- .../tests/fixtures/planner/expected-plan.json | 4 +- .../fixtures/planner/expected-report.json | 58 ++++++-- .../sie_placement/tests/test_availability.py | 6 +- packages/sie_placement/tests/test_levers.py | 2 +- packages/sie_placement/tests/test_packing.py | 6 +- packages/sie_placement/tests/test_planner.py | 139 ++++++++++++++++-- 8 files changed, 229 insertions(+), 46 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 28fcd22cc..ec374571d 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -99,6 +99,16 @@ refers to its lane by that name. Two plan lanes may carry the same three tokens, and so name one chart lane, which cannot give them separate envelopes or concurrency; the planner's report lists every such chart lane. +The planner gives each lane that pins at least one identity a named queue pool +of its own, since the chart carries a lane's pinned set on a static queue pool +and the default queue pool carries no pins. A lane the current plan already +puts on a named queue pool keeps it, so planning again leaves the pool as it +is. Otherwise the pool is derived from the lane's `name`: the name itself, cut +to 63 characters with a digest of the name appended when it is longer or when +another lane of the plan, `default`, or the parameters' `queue_pool` already +has it. A lane on which every identity is lazy-resident or cold is on the +parameters' `queue_pool`. Identities that share a lane share its queue pool. + `fallbackCells` is an optional ordered candidate list: cells, after `cell`, that the lane's containers may be placed on when `cell` cannot be placed. No fallback cell repeats `cell` or another fallback cell. A realized card is @@ -220,7 +230,7 @@ choice, such as its own previous plan, reports no change. | `buffer_containers` | The declared warm buffer above a lane's floor, `0` by default | | `hold_budget_s` | The longest a request for a cold lane is held for it before it is refused, `0` by default. An identity whose `comeAliveP95S` is at most this is served by holding its requests, and avoids unreliable GPU products | | `hours_per_month` | The hours a monthly figure covers, `730` by default | -| `queue_pool`, `bundle`, `bundles` | Every lane's queue pool, and each identity's bundle: its `bundles` entry, otherwise `bundle` | +| `queue_pool`, `bundle`, `bundles` | The queue pool of every lane that pins no identity, and each identity's bundle: its `bundles` entry, otherwise `bundle` | | `unshared` | Identities that never share a lane: none is packed or made a lazy resident beside another identity, and no other identity joins a lane one of them is on. Empty by default | ### Selection @@ -256,8 +266,7 @@ document: `comeAliveP95S` of `0` admits `pinned` only. 4. A cell also offers `lazy-resident` on each host lane with its cell, container shape and concurrency. A host lane is a lane of the current plan - with a `floorMin` of at least 1, on the parameters' queue pool and the - identity's bundle, on which another identity whose `comeAliveP95S` is `0` + with a `floorMin` of at least 1, on the identity's bundle, on which another identity whose `comeAliveP95S` is `0` stays pinned in this run, on its current placement or kept there. Those identities are placed first: a bound of `0` admits no cold or lazy option, so no other choice moves them. The identity's own current lane is also a @@ -288,8 +297,8 @@ document: The current placement is the identity's current lane when that lane has the option's cell and fallback cells, container shape and concurrency, the -identity's warmth is the option's, and the lane's `queuePool` and `bundle` are -the ones the parameters give the identity; a `lazy-resident` option must also be +identity's warmth is the option's, and the lane's `bundle` is the one the +parameters give the identity; a `lazy-resident` option must also be hosted by that lane, and a `pinned` option on a lane where other identities are pinned must stand on a co-residency receipt beside them. A current placement that no option matches lacks evidence: a receipt at its lane's cell, container @@ -337,7 +346,7 @@ plan total does not fall. 4. Packing. The planner visits every placed identity that is alone on its lane and is neither a generation nor an `unshared` identity, in decreasing order of its lane's floor cost, ties broken by identity. For each it tries the - other lanes of the plan on its queue pool and bundle that keep a floor and + other lanes of the plan on its bundle that keep a floor and host a pinned identity, where it has a receipt at the lane's cell, container shape and concurrency: first the lanes on which an identity whose `comeAliveP95S` is `0` is pinned, since nothing moves that identity, then in diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index b24c2e805..6db08526f 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -67,6 +67,8 @@ PLACEMENT_FAILURE_FIELD = "placementFailureRate" MEMORY_HEADROOM_FRACTION = 0.8 COLD_WARMTHS = (Warmth.COLD_WITH_BOUND, Warmth.SNAPSHOT_COLD) +DEFAULT_QUEUE_POOL = "default" +QUEUE_POOL_MAX_LENGTH = 63 _SLUG = re.compile(r"[^a-z0-9]+") _LEVEL_FIELDS = { @@ -100,7 +102,7 @@ class PlannerParameters: buffer_containers: int = 0 hold_budget_s: int = 0 hours_per_month: float = HOURS_PER_MONTH - queue_pool: str = "default" + queue_pool: str = DEFAULT_QUEUE_POOL bundle: str = "default" bundles: Mapping[str, str] = field(default_factory=dict) unshared: frozenset[str] = frozenset() @@ -383,6 +385,18 @@ def _digest(value: str, width: int = 8) -> str: return hashlib.sha256(value.encode()).hexdigest()[:width] +def _named_queue_pool(lane: str, taken: set[str]) -> str: + base = _slug(lane) or "lane" + candidates = [base] if len(base) <= QUEUE_POOL_MAX_LENGTH else [] + candidates.extend( + f"{base[: QUEUE_POOL_MAX_LENGTH - width - 1].rstrip('-')}-{_digest(lane, width)}" for width in (8, 16) + ) + name = next((candidate for candidate in candidates if candidate not in taken), None) + if name is None: + raise PlannerInputError(f"queue pool name for lane {lane!r} is already taken") + return name + + def _refusal_reason(cell: Cell, prices: Mapping[str, MachineProfilePrice]) -> str | None: price = prices.get(cell.machine_profile) if price is None: @@ -667,7 +681,6 @@ def is_current( return False return ( entry.warmth is warmth - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and lane.cells == (receipt.cell, *fallbacks) and _runs_on(receipt, lane) @@ -798,7 +811,7 @@ def keepable(self, identity: str) -> tuple[PlanIdentity, PlanLane] | None: if current is None: return None lane = current[1] - if lane.queue_pool != self.parameters.queue_pool or lane.bundle != self.parameters.bundle_of(identity): + if lane.bundle != self.parameters.bundle_of(identity): return None return current @@ -828,7 +841,6 @@ def lazy_hosts(self, identity: str) -> tuple[PlanLane, ...]: and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and not self.hosts_solitary((*self.current_pinned[name], *self.current_lazy[name])) - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == bundle ) @@ -1461,6 +1473,21 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: envelope=envelope_for(option.warmth, self.parameters), ) + def queue_pools(self, lanes: Iterable[str], identities: Iterable[PlanIdentity]) -> dict[str, str]: + pools = dict.fromkeys(lanes, self.parameters.queue_pool) + pinning = sorted({entry.lane for entry in identities if entry.warmth is Warmth.PINNED}) + taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool} + for name in pinning: + current = self.current_lanes.get(name) + if current is not None and current.queue_pool not in taken: + pools[name] = current.queue_pool + taken.add(current.queue_pool) + for name in pinning: + if pools[name] == self.parameters.queue_pool: + pools[name] = _named_queue_pool(name, taken) + taken.add(pools[name]) + return pools + def within_ceiling(self, lane: PlanLane) -> PlanLane: ceiling = self.parameters.max_containers if lane.max_containers <= ceiling: @@ -1545,9 +1572,10 @@ def assemble(self, choices: Mapping[str, _Choice]) -> _Layout: PlanIdentity(identity=identity, lane=names[identity], warmth=option.warmth) for identity, option in placed.items() ) + pools = self.queue_pools(lanes, identities) plan = Plan( placement=self.parameters.placement, - lanes=tuple(lanes[name] for name in sorted(lanes)), + lanes=tuple(replace(lanes[name], queue_pool=pools[name]) for name in sorted(lanes)), identities=tuple(sorted(identities, key=lambda entry: entry.identity)), ) receipts: dict[str, Receipt | None] = {identity: option.pricing_receipt for identity, option in placed.items()} @@ -1639,7 +1667,6 @@ def pack_targets(self, identity: str, layout: _Layout) -> list[tuple[PlanLane, f and lane.envelope.floor_min >= PINNED_FLOOR and not lane.fallback_cells and lane.container_shape.gpu_count == 1 - and lane.queue_pool == self.parameters.queue_pool and lane.bundle == self.parameters.bundle_of(identity) and pinned[lane.name] and not self.hosts_solitary((*pinned[lane.name], *lazy[lane.name])) diff --git a/packages/sie_placement/tests/fixtures/planner/expected-plan.json b/packages/sie_placement/tests/fixtures/planner/expected-plan.json index 303a4ed79..32c31b3be 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-plan.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-plan.json @@ -25,7 +25,7 @@ }, { "name": "default-l4-default", - "queuePool": "default", + "queuePool": "default-l4-default", "bundle": "default", "cell": { "machineProfile": "l4", @@ -65,7 +65,7 @@ }, { "name": "qwen-qwen3-embedding-4b-l4", - "queuePool": "default", + "queuePool": "qwen-qwen3-embedding-4b-l4", "bundle": "default", "cell": { "machineProfile": "l4", diff --git a/packages/sie_placement/tests/fixtures/planner/expected-report.json b/packages/sie_placement/tests/fixtures/planner/expected-report.json index 226647ae1..44b50fb0c 100644 --- a/packages/sie_placement/tests/fixtures/planner/expected-report.json +++ b/packages/sie_placement/tests/fixtures/planner/expected-report.json @@ -327,18 +327,7 @@ } ], "capacity": [], - "sharedChartLanes": [ - { - "queuePool": "default", - "machineProfile": "l4", - "bundle": "default", - "lanes": [ - "baai-bge-m3-sparse-l4", - "default-l4-default", - "qwen-qwen3-embedding-4b-l4" - ] - } - ], + "sharedChartLanes": [], "measurementRequests": [ { "identity": "Qwen/Qwen3-Reranker-0.6B", @@ -632,11 +621,54 @@ } } }, + { + "name": "default-l4-default", + "before": { + "name": "default-l4-default", + "queuePool": "default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + }, + "after": { + "name": "default-l4-default", + "queuePool": "default-l4-default", + "bundle": "default", + "cell": { + "machineProfile": "l4", + "gpuProduct": "NVIDIA-L4" + }, + "containerShape": { + "gpuCount": 1, + "cpuCores": 4, + "memoryMiB": 16384 + }, + "concurrency": 16, + "maxContainers": 4, + "envelope": { + "floorMin": 1, + "floorMax": 2 + } + } + }, { "name": "qwen-qwen3-embedding-4b-l4", "after": { "name": "qwen-qwen3-embedding-4b-l4", - "queuePool": "default", + "queuePool": "qwen-qwen3-embedding-4b-l4", "bundle": "default", "cell": { "machineProfile": "l4", diff --git a/packages/sie_placement/tests/test_availability.py b/packages/sie_placement/tests/test_availability.py index 98a0a683d..b1c21433b 100644 --- a/packages/sie_placement/tests/test_availability.py +++ b/packages/sie_placement/tests/test_availability.py @@ -191,9 +191,13 @@ def lane(name, cell=L4, *, fallbacks=(), floor=1, buffer=0) -> PlanLane: def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) diff --git a/packages/sie_placement/tests/test_levers.py b/packages/sie_placement/tests/test_levers.py index aec54c2bf..8721c77a5 100644 --- a/packages/sie_placement/tests/test_levers.py +++ b/packages/sie_placement/tests/test_levers.py @@ -125,7 +125,7 @@ def shared_lane(*identities): lanes=( PlanLane( name="shared", - queue_pool="default", + queue_pool="shared", bundle="default", cell=L4, container_shape=SHAPE, diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index b3287c54f..ecc2f36d5 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -197,9 +197,13 @@ def lane(name, cell=L4, *, floor=1, shape=SHAPE) -> PlanLane: def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index e0d78d211..f5a73fcb3 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -239,7 +239,7 @@ def single_lane_plan( lanes=( PlanLane( name=name, - queue_pool="default", + queue_pool=name if warmth is Warmth.PINNED else "default", bundle="default", cell=cell, container_shape=SHAPE, @@ -271,9 +271,13 @@ def plan_lane(name, *, cell=L4, floor=1, bundle="default", queue_pool="default") def plan_of(lanes, residents) -> Plan: + pinning = {name for name, warmth in residents.values() if warmth is Warmth.PINNED} return Plan( placement="test", - lanes=tuple(lanes), + lanes=tuple( + replace(lane, queue_pool=lane.name) if lane.name in pinning and lane.queue_pool == "default" else lane + for lane in lanes + ), identities=tuple( PlanIdentity(identity=identity, lane=name, warmth=warmth) for identity, (name, warmth) in sorted(residents.items()) @@ -1331,9 +1335,11 @@ def test_a_lazy_resident_left_alone_on_its_lane_stays_when_nothing_is_cheaper_th assert result.report.monthly_cost == pytest.approx(MONTH) -@pytest.mark.parametrize("shared", [plan_lane("shared", bundle="sglang"), plan_lane("shared", queue_pool="batch")]) -def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): - current = plan_of([plan_lane("own"), shared], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}) +def test_a_lazy_resident_needs_a_lane_of_its_bundle(): + current = plan_of( + [plan_lane("own"), plan_lane("shared", bundle="sglang")], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, current=current) assert {entry.identity: entry.lane for entry in result.plan.identities} == { ANCHOR: "org-anchor-l4", @@ -1343,6 +1349,20 @@ def test_a_lazy_resident_needs_a_lane_of_its_pool_and_bundle(shared): assert all(request.lane != "shared" for request in result.report.measurement_requests) +def test_a_lazy_resident_joins_a_pinned_lane_on_its_named_queue_pool(): + current = plan_of( + [plan_lane("own"), plan_lane("shared", queue_pool="batch")], + {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)}, + ) + hourly = {IDENTITY: {3: 90000}} + result = run(resident_levels(), RESIDENT_RECEIPTS, LOADS, prices=L4_ONLY, hourly=hourly, current=current) + assert [(entry.identity, entry.lane, entry.warmth) for entry in result.plan.identities] == [ + (ANCHOR, "shared", Warmth.PINNED), + (IDENTITY, "shared", Warmth.LAZY_RESIDENT), + ] + assert result.plan.lanes == (current.lanes[1],) + + def test_a_lazy_resident_needs_a_never_cold_neighbour_on_a_current_lane_and_on_a_packed_one(): current = plan_of( [plan_lane("own"), plan_lane("shared")], {ANCHOR: ("shared", Warmth.PINNED), IDENTITY: ("own", Warmth.PINNED)} @@ -1427,17 +1447,25 @@ def test_a_placement_that_moves_leaves_its_shared_lane_to_the_kept_neighbour(): @pytest.mark.parametrize( - ("overrides", "moved"), - [({"bundles": {IDENTITY: "sglang"}}, ("default", "sglang")), ({"queue_pool": "batch"}, ("batch", "default"))], + ("overrides", "expected"), + [ + ( + {"bundles": {IDENTITY: "sglang"}}, + {"org-model-l4": ("org-model-l4", "sglang"), "shared": ("shared", "default")}, + ), + ({"queue_pool": "batch"}, {"shared": ("shared", "default")}), + ], ) -def test_a_new_queue_pool_or_bundle_moves_the_identity_off_its_current_lane(overrides, moved): +def test_a_new_bundle_moves_the_identity_off_its_current_lane_and_a_new_default_queue_pool_does_not( + overrides, expected +): current = single_lane_plan(others=("org/kept",), max_containers=4) service_levels = levels( (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) ) result = run(service_levels, [receipt()], prices=L4_ONLY, current=current, **overrides) lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} - assert lanes == {"org-model-l4": moved, "shared": ("default", "default")} + assert lanes == expected def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): @@ -1455,7 +1483,9 @@ def test_removing_one_of_two_colliding_identities_keeps_the_other_lane_name(): def test_plan_lanes_that_name_one_chart_lane_are_reported(): service_levels = levels(("org/a", WARM), ("org/b", WARM), ("org/c", WARM)) receipts = [receipt(identity=identity) for identity in ("org/a", "org/b", "org/c")] - result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/c": "sglang"}) + boots = [boot(identity=identity) for identity in ("org/a", "org/b", "org/c")] + result = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/c": "sglang"}) + assert {entry.warmth for entry in result.plan.identities} == {Warmth.COLD_WITH_BOUND} (shared,) = result.report.shared_chart_lanes assert shared.to_dict() == { "queuePool": "default", @@ -1463,8 +1493,77 @@ def test_plan_lanes_that_name_one_chart_lane_are_reported(): "bundle": "default", "lanes": ["org-a-l4", "org-b-l4"], } - apart = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "a", "org/b": "b"}) + apart = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/a": "a", "org/b": "b"}) assert apart.report.shared_chart_lanes == () + pinned = run(service_levels, receipts, prices=L4_ONLY) + assert {entry.warmth for entry in pinned.plan.identities} == {Warmth.PINNED} + assert pinned.report.shared_chart_lanes == () + + +def test_each_lane_that_pins_an_identity_gets_its_own_named_queue_pool(): + long = "org/" + "x" * 70 + never_cold = ServiceLevel(come_alive_p95_s=0, warm_p99_ms=ANY_P99_MS) + service_levels = levels(("org/a", never_cold), ("org/b", never_cold), (long, never_cold), ("org/c", WARM)) + receipts = [ + receipt(identity="org/a"), + receipt(identity="org/b", cell=A100), + receipt(identity=long, cell=A100), + receipt(identity="org/c"), + ] + result = run(service_levels, receipts, [boot(identity="org/c")]) + warmths = {entry.identity: (entry.lane, entry.warmth) for entry in result.plan.identities} + long_lane = warmths[long][0] + assert warmths == { + "org/a": ("org-a-l4", Warmth.PINNED), + "org/b": ("org-b-a100-80gb", Warmth.PINNED), + long: (long_lane, Warmth.PINNED), + "org/c": ("org-c-l4", Warmth.COLD_WITH_BOUND), + } + pools = {lane.name: lane.queue_pool for lane in result.plan.lanes} + long_pool = f"{long_lane[:54]}-{hashlib.sha256(long_lane.encode()).hexdigest()[:8]}" + assert pools == { + "org-a-l4": "org-a-l4", + "org-b-a100-80gb": "org-b-a100-80gb", + long_lane: long_pool, + "org-c-l4": "default", + } + assert len(long_pool) == 63 + again = run(service_levels, receipts, [boot(identity="org/c")], current=result.plan) + assert again.plan == result.plan + assert again.report.unchanged + + +def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_the_default_pool(): + named = single_lane_plan(name="kept", identity="org/kept") + named_lane = replace(named.lanes[0], queue_pool="org-model-l4") + unnamed = single_lane_plan(name="org-model-l4") + unnamed_lane = replace(unnamed.lanes[0], queue_pool="default") + current = Plan( + placement="test", + lanes=(unnamed_lane, named_lane), + identities=(*unnamed.identities, *named.identities), + ) + service_levels = levels( + (IDENTITY, ServiceLevel(come_alive_p95_s=0)), ("org/kept", ServiceLevel(come_alive_p95_s=0)) + ) + result = run(service_levels, [receipt()], prices=L4_ONLY, current=current) + assert {entry.identity: entry.lane for entry in result.plan.identities} == { + IDENTITY: "org-model-l4", + "org/kept": "kept", + } + digest = hashlib.sha256(b"org-model-l4").hexdigest()[:8] + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == { + "kept": "org-model-l4", + "org-model-l4": f"org-model-l4-{digest}", + } + assert [(change.name, change.after.queue_pool) for change in result.report.lane_changes] == [ + ("org-model-l4", f"org-model-l4-{digest}") + ] + moved = run(service_levels, [receipt()], prices=L4_ONLY, current=current, queue_pool="org-model-l4") + assert {lane.name: lane.queue_pool for lane in moved.plan.lanes} == { + "kept": "kept", + "org-model-l4": f"org-model-l4-{digest}", + } def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): @@ -1489,18 +1588,19 @@ def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): assert {lane.name: lane for lane in result.plan.lanes}[f"org-model-l4-{digest[:8]}"] == second.lanes[0] -def test_bundles_and_the_queue_pool_come_from_the_parameters(): +def test_bundles_and_the_default_queue_pool_come_from_the_parameters(): service_levels = levels(("org/embed", WARM), ("org/generate", WARM)) result = run( service_levels, [receipt(identity="org/embed"), receipt(identity="org/generate")], + [boot(identity="org/embed")], prices=L4_ONLY, queue_pool="batch", bundle="default", bundles={"org/generate": "sglang"}, ) lanes = {lane.name: (lane.queue_pool, lane.bundle) for lane in result.plan.lanes} - assert lanes == {"org-embed-l4": ("batch", "default"), "org-generate-l4": ("batch", "sglang")} + assert lanes == {"org-embed-l4": ("batch", "default"), "org-generate-l4": ("org-generate-l4", "sglang")} def test_equal_costs_prefer_the_current_placement(): @@ -1684,8 +1784,13 @@ def test_the_estate_report_states_its_findings(): assert shared.monthly_cost == pytest.approx(dense.monthly_cost) assert shared.unpriced_identities == ("intfloat/e5-base-v2",) assert report.unpriced_lanes == ("default-l4-default",) - (chart_lane,) = report.shared_chart_lanes - assert chart_lane.lanes == ("baai-bge-m3-sparse-l4", "default-l4-default", "qwen-qwen3-embedding-4b-l4") + assert report.shared_chart_lanes == () + assert {lane.name: lane.queue_pool for lane in plan(**estate()).plan.lanes} == { + "baai-bge-m3-sparse-l4": "default", + "default-l4-default": "default-l4-default", + "qwen-qwen3-8b-a100-80gb": "default", + "qwen-qwen3-embedding-4b-l4": "qwen-qwen3-embedding-4b-l4", + } current = estate()["current_plan"] assert sum(lane.envelope.floor_min for lane in plan(**estate()).plan.lanes) == 1 + sum( lane.envelope.floor_min for lane in current.lanes @@ -1942,9 +2047,11 @@ def test_random_current_estates_keep_what_no_measured_option_beats(seed): assert Plan.from_dict(result.plan.to_dict()) == result.plan before = {lane.name: lane for lane in current.lanes} after = {lane.name: lane for lane in result.plan.lanes} + pinning = {entry.lane for entry in result.plan.identities if entry.warmth is Warmth.PINNED} for name, lane in after.items(): if name in before: - assert lane == capped(before[name], 4) + assert replace(lane, queue_pool=before[name].queue_pool) == capped(before[name], 4) + assert lane.queue_pool == (name if name in pinning else "default") placements = {entry.identity: entry for entry in result.plan.identities} never_cold = {entry.identity for entry in service_levels.identities if entry.level.come_alive_p95_s == 0} anchored = { From 2601bb61b8d7af4a3203bd6515e918dfa160c9b1 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:19:58 +0100 Subject: [PATCH 05/10] test(placement): hold a shared lane to the shared quota at its envelope maximum - A shared lane holds its container ceiling once against the total and its GPU type, and its envelope maximum once against the shared quota, so a packing fits a quota the separate lanes' envelope maxima would exceed. --- packages/sie_placement/tests/test_packing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index ecc2f36d5..2129301cd 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -655,11 +655,11 @@ def test_an_unshared_identity_is_never_packed_and_its_lane_hosts_no_one_else(uns assert placements(shared)[GUEST] == ("host", Warmth.LAZY_RESIDENT) -def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): +def test_a_shared_lane_holds_its_container_ceiling_and_envelope_maximum_once_against_every_ceiling(): capacity = Capacity( total=4, types=(TypeCeiling(gpu_product=L4.gpu_product, accelerators=4),), - shared_quota=SharedQuota(accelerators=6, held_elsewhere=2), + shared_quota=SharedQuota(accelerators=3, held_elsewhere=2), ) service_levels = levels(**{HOST: 0, GUEST: 60}) result = run(service_levels, host_and_guest(), [load(GUEST)], capacity=capacity) @@ -667,15 +667,15 @@ def test_a_shared_lane_holds_its_container_ceiling_once_against_every_ceiling(): assert [(use.accelerators, use.peak_accelerators, use.lanes) for use in result.report.capacity] == [ (1, 4, ("org-host-l4",)), (1, 4, ("org-host-l4",)), - (3, 6, ("org-host-l4",)), + (3, None, ("org-host-l4",)), ] with pytest.raises(CapacityExceededError) as caught: run(service_levels, host_and_guest(guest_footprint=8000), [load(GUEST)], capacity=capacity) assert [use.describe() for use in caught.value.exceeded] == [ "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, above the total ceiling of 4", "the container ceilings of org-guest-l4, org-host-l4 hold 8 NVIDIA-L4 accelerators, above the type ceiling of 4", - "the container ceilings of org-guest-l4, org-host-l4 hold 8 accelerators, " - "which with 2 held elsewhere exceed the shared quota of 6", + "the envelope maxima of org-guest-l4, org-host-l4 hold 2 accelerators, " + "which with 2 held elsewhere exceed the shared quota of 3", ] From b8ff7d86e88b00f337e6b0506f164308e699e826 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:31:07 +0100 Subject: [PATCH 06/10] fix(placement): check memory sizes as integers in documents built in code - A receipt's footprintMiB and workingMemoryMiB, a boot row's footprintMiB and a machine profile's gpuMemoryMiB go through the integer check, so a document built in code refuses booleans and floats as parsing does. --- .../sie_placement/src/sie_placement/schema.py | 8 +++--- packages/sie_placement/tests/test_schema.py | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/src/sie_placement/schema.py b/packages/sie_placement/src/sie_placement/schema.py index ba9b8404a..4909e13c2 100644 --- a/packages/sie_placement/src/sie_placement/schema.py +++ b/packages/sie_placement/src/sie_placement/schema.py @@ -644,8 +644,8 @@ class Receipt: def __post_init__(self) -> None: _require_identifier("identity", self.identity) _require_count("concurrency", self.concurrency, 1) - _require_at_least("footprintMiB", self.footprint_mib, 1) - _require_at_least("workingMemoryMiB", self.working_memory_mib, 0) + _require_count("footprintMiB", self.footprint_mib, 1) + _require_count("workingMemoryMiB", self.working_memory_mib, 0) for index, resident in enumerate(self.co_residents): _require_identifier(f"coResidents[{index}]", resident) if resident == self.identity: @@ -742,7 +742,7 @@ def __post_init__(self) -> None: raise SchemaError("bootSeconds", "is required for an attempt that was placed") _require_at_least("placementWaitS", self.placement_wait_s, 0) _require_at_least("bootSeconds", self.boot_seconds, 0) - _require_at_least("footprintMiB", self.footprint_mib, 1) + _require_count("footprintMiB", self.footprint_mib, 1) if self.outcome is not BootOutcome.SUCCEEDED: _require_absent("footprintMiB", self.footprint_mib, "for an attempt that did not succeed") @@ -798,7 +798,7 @@ def __post_init__(self) -> None: _require_at_least("acceleratorPerHour", self.accelerator_per_hour, 0) _require_at_least("corePerHour", self.core_per_hour, 0) _require_at_least("memoryGiBPerHour", self.memory_gib_per_hour, 0) - _require_at_least("gpuMemoryMiB", self.gpu_memory_mib, 1) + _require_count("gpuMemoryMiB", self.gpu_memory_mib, 1) @classmethod def from_dict(cls, data: object, path: str = ROOT_PATH) -> Self: diff --git a/packages/sie_placement/tests/test_schema.py b/packages/sie_placement/tests/test_schema.py index 3aa73bf9d..6248bab36 100644 --- a/packages/sie_placement/tests/test_schema.py +++ b/packages/sie_placement/tests/test_schema.py @@ -4,6 +4,7 @@ import json import math from collections.abc import Callable +from dataclasses import replace from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -190,6 +191,17 @@ def test_numbers_built_in_code_must_be_numbers(value, described): (lambda value: Capacity(total=value), "total"), (lambda value: TypeCeiling(gpu_product="NVIDIA-L4", accelerators=value), "accelerators"), (lambda value: SharedQuota(accelerators=8, held_elsewhere=value), "heldElsewhere"), + ( + lambda value: MachineProfilePrice( + name="l4", + gpu_product="NVIDIA-L4", + accelerator_per_hour=0, + core_per_hour=0, + memory_gib_per_hour=0, + gpu_memory_mib=value, + ), + "gpuMemoryMiB", + ), ], ) @pytest.mark.parametrize(("value", "described"), [(1.5, "number 1.5"), (2.0, "number 2.0"), (True, "boolean true")]) @@ -566,6 +578,21 @@ def test_boot_evidence_records_the_footprint_of_a_load_that_succeeded(): ) +@pytest.mark.parametrize( + ("parse", "name", "field"), + [ + (Receipt.from_dict, "receipt-encode.json", "footprint_mib"), + (Receipt.from_dict, "receipt-encode.json", "working_memory_mib"), + (BootEvidence.from_dict, "boot-evidence.json", "footprint_mib"), + ], +) +def test_memory_built_in_code_must_be_integers(parse, name, field): + document = parse(load(name)) + with pytest.raises(SchemaError) as caught: + replace(document, **{field: 1.5}) + assert caught.value.message == "must be an integer, got number 1.5" + + def test_machine_profiles_may_state_the_memory_of_their_card(): data = load("price-list.json") data["machineProfiles"][0]["gpuMemoryMiB"] = 23034 From cb0ed0a704c5ca26e82740b419d3a2f350665b28 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:32:50 +0100 Subject: [PATCH 07/10] docs(placement): say a derived queue pool is the slug of the lane name - A lane whose current plan puts it on the default pool and is named Shared_Lane derives the pool shared-lane, which a test now pins. --- packages/sie_placement/README.md | 13 ++++++++----- packages/sie_placement/tests/test_planner.py | 7 +++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 0aff76c9f..f995bd7ae 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -103,11 +103,14 @@ The planner gives each lane that pins at least one identity a named queue pool of its own, since the chart carries a lane's pinned set on a static queue pool and the default queue pool carries no pins. A lane the current plan already puts on a named queue pool keeps it, so planning again leaves the pool as it -is. Otherwise the pool is derived from the lane's `name`: the name itself, cut -to 63 characters with a digest of the name appended when it is longer or when -another lane of the plan, `default`, or the parameters' `queue_pool` already -has it. A lane on which every identity is lazy-resident or cold is on the -parameters' `queue_pool`. Identities that share a lane share its queue pool. +is. Otherwise the pool is derived from the lane's `name`: its slug, which +lowercases the name, replaces each run of characters outside `a-z` and `0-9` +with a hyphen and drops leading and trailing hyphens, or `lane` when nothing +remains. The slug is cut to 63 characters with a digest of the name appended +when it is longer or when another lane of the plan, `default`, or the +parameters' `queue_pool` already has it. A lane on which every identity is +lazy-resident or cold is on the parameters' `queue_pool`. Identities that share +a lane share its queue pool. `fallbackCells` is an optional ordered candidate list: cells, after `cell`, that the lane's containers may be placed on when `cell` cannot be placed. No diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index f5a73fcb3..9d06386bc 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1566,6 +1566,13 @@ def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_ } +def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): + current = single_lane_plan(name="Shared_Lane") + current = replace(current, lanes=(replace(current.lanes[0], queue_pool="default"),)) + result = run(levels((IDENTITY, ServiceLevel(come_alive_p95_s=0))), [receipt()], prices=L4_ONLY, current=current) + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == {"Shared_Lane": "shared-lane"} + + def test_a_derived_lane_name_never_takes_a_kept_lane_named_with_its_digest(): digest = hashlib.sha256(IDENTITY.encode()).hexdigest() first = single_lane_plan(name="org-model-l4", identity="org/kept") From 8676b86ddfa7ef7f679d63c30a97a98bdfbb4a53 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:44:36 +0100 Subject: [PATCH 08/10] fix(placement): compare kept queue pools without regard to case - A pinning lane keeps its current named queue pool only when no other lane keeps it and it is neither default nor the parameters' queue_pool, each compared in lowercase as the chart canonicalizes lane tokens. A current pool spelled Default, or a second lane on Batch and batch, gets a derived pool instead. --- packages/sie_placement/README.md | 4 +++- .../src/sie_placement/planner.py | 6 +++--- packages/sie_placement/tests/test_planner.py | 21 ++++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 744c44b95..7de99ff95 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -104,7 +104,9 @@ The planner gives each lane that pins at least one identity a named queue pool of its own, since the chart carries a lane's pinned set on a static queue pool and the default queue pool carries no pins. A lane the current plan already puts on a named queue pool keeps it, so planning again leaves the pool as it -is. Otherwise the pool is derived from the lane's `name`: its slug, which +is, unless the pool is `default`, the parameters' `queue_pool` or a pool +another lane keeps, compared without regard to case as the chart compares them. +Otherwise the pool is derived from the lane's `name`: its slug, which lowercases the name, replaces each run of characters outside `a-z` and `0-9` with a hyphen and drops leading and trailing hyphens, or `lane` when nothing remains. The slug is cut to 63 characters with a digest of the name appended diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 68bff74e0..33df139e6 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -1475,12 +1475,12 @@ def placed_lane(self, name: str, identity: str, option: _Option) -> PlanLane: def queue_pools(self, lanes: Iterable[str], identities: Iterable[PlanIdentity]) -> dict[str, str]: pools = dict.fromkeys(lanes, self.parameters.queue_pool) pinning = sorted({entry.lane for entry in identities if entry.warmth is Warmth.PINNED}) - taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool} + taken = {DEFAULT_QUEUE_POOL, self.parameters.queue_pool.lower()} for name in pinning: current = self.current_lanes.get(name) - if current is not None and current.queue_pool not in taken: + if current is not None and current.queue_pool.lower() not in taken: pools[name] = current.queue_pool - taken.add(current.queue_pool) + taken.add(current.queue_pool.lower()) for name in pinning: if pools[name] == self.parameters.queue_pool: pools[name] = _named_queue_pool(name, taken) diff --git a/packages/sie_placement/tests/test_planner.py b/packages/sie_placement/tests/test_planner.py index 057d1748b..1cc876fab 100644 --- a/packages/sie_placement/tests/test_planner.py +++ b/packages/sie_placement/tests/test_planner.py @@ -1566,6 +1566,24 @@ def test_a_replan_keeps_a_named_queue_pool_and_derives_one_for_a_pinned_lane_on_ } +def test_a_kept_queue_pool_is_compared_without_regard_to_case(): + kept = [single_lane_plan(name=name, identity=f"org/{name}") for name in ("first", "second", "third")] + pools = ("Batch", "batch", "Default") + current = Plan( + placement="test", + lanes=tuple(replace(entry.lanes[0], queue_pool=pool) for entry, pool in zip(kept, pools, strict=True)), + identities=tuple(identity for entry in kept for identity in entry.identities), + ) + pinned = ServiceLevel(come_alive_p95_s=0) + service_levels = levels(*((f"org/{name}", pinned) for name in ("first", "second", "third"))) + result = run(service_levels, [], prices=L4_ONLY, current=current) + assert {lane.name: lane.queue_pool for lane in result.plan.lanes} == { + "first": "Batch", + "second": "second", + "third": "third", + } + + def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): current = single_lane_plan(name="Shared_Lane") current = replace(current, lanes=(replace(current.lanes[0], queue_pool="default"),)) @@ -1576,7 +1594,8 @@ def test_a_derived_queue_pool_is_the_slug_of_the_lane_name(): def test_plan_lanes_whose_tokens_differ_only_in_case_name_one_chart_lane(): service_levels = levels(("org/a", WARM), ("org/b", WARM)) receipts = [receipt(identity=identity) for identity in ("org/a", "org/b")] - result = run(service_levels, receipts, prices=L4_ONLY, bundles={"org/a": "Default"}) + boots = [boot(identity=identity) for identity in ("org/a", "org/b")] + result = run(service_levels, receipts, boots, prices=L4_ONLY, bundles={"org/a": "Default"}) assert {lane.name: lane.bundle for lane in result.plan.lanes} == {"org-a-l4": "Default", "org-b-l4": "default"} (shared,) = result.report.shared_chart_lanes assert shared.to_dict() == { From 5f3d1a653020d070cae81ae5699b8d4e0e46b46f Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 10:54:56 +0100 Subject: [PATCH 09/10] docs(placement): say when the planner requests footprints and what a profile without card memory blocks --- packages/sie_placement/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 7de99ff95..24606bb52 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -189,10 +189,15 @@ it had before. Where the identities that join a lane do not fit it together, they are admitted first fit in decreasing order of footprint, ties broken by identity, and the others are decided again without that lane. -Where a resident's footprint or working memory is not on file, the planner -asks that resident for it, under its own identity and warmth at the lane, in a -`footprint` measurement request. A lane on a profile without `gpuMemoryMiB` -hosts no identity beside another, and the report names that profile. +Where an identity would join a lane and a resident's footprint or working +memory is not on file, the planner asks that resident for it, under its own +identity and warmth at the lane, in a `footprint` measurement request. For a +lane the plan keeps, it asks only when a lazy resident is placed there. A kept +lane whose pinned residents have no footprint on file gets no `footprint` +request, so the report does not name every unmeasured shared lane. A profile +without `gpuMemoryMiB` takes no new identity beside another, and the report +names that profile. A lane the plan already shares on such a profile stays +shared. ## Envelope From c35ea60ac3fd41de0172ea73f5a42bdbea0f593a Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Fri, 6 Nov 2026 11:14:55 +0100 Subject: [PATCH 10/10] fix(placement): size a shared lane from its residents' demand together - A lane with a floor and several priced residents now costs the containers each hour needs for their requests together, each resident's requests taking the share of a container its priced receipt's rate gives them. Pricing each resident's demand against the floor on its own counted a container that both residents need as free, so a packing could look cheaper than the lane it makes. - A packing after which the lane's residents together need more containers than its maxContainers in some hour is not made, so two residents that each fill a one-container ceiling are not packed onto it. --- packages/sie_placement/README.md | 23 ++++++----- .../src/sie_placement/planner.py | 38 +++++++++++++++++-- packages/sie_placement/tests/test_packing.py | 27 +++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/packages/sie_placement/README.md b/packages/sie_placement/README.md index 24606bb52..f3e3701a1 100644 --- a/packages/sie_placement/README.md +++ b/packages/sie_placement/README.md @@ -375,7 +375,9 @@ plan total does not fall. otherwise pinned, which needs every pinned identity's co-residency receipt beside the others to meet its warm bounds. A move that would lower the total but lacks a footprint, a working memory, a load or a co-residency receipt is - not made: it is a measurement request whose `savingBound` is that fall. + not made: it is a measurement request whose `savingBound` is that fall. A + move after which the lane's residents together need more containers than + its `maxContainers` in some hour is not made either. 5. Steps 1 and 3 run again for the lanes packing changed. ### Pricing @@ -393,14 +395,17 @@ demand window to `hours_per_month`. every request: the active floor for `cooldown_s`, at most the whole hour. The active floor is `fallback_floor` capped by the ceiling, and at least one. -A lane costs its floor all month once, whoever it hosts, plus what each -resident's demand needs beyond the floor: on a lane with a floor, the -containers an hour needs above it, and on a zero-floor lane, the work and the -tails a cold warmth pays. A placed identity's demand is priced at its chosen -receipt, and a kept identity's at the slowest receipt it has for its lane's -cells, container shape and concurrency, under that lane's own ceiling. A lane -with fallback cells costs the hourly rate of its dearest cell, and a candidate -list's demand is priced at the slower of its two receipts. +A lane costs its floor all month once, whoever it hosts, plus what its +residents' demand needs beyond the floor. On a lane with a floor, that is the +containers an hour needs above it for the residents together: each resident's +requests in the hour take the share of a container that its priced receipt's +rate gives them, and the shares are summed, since residents scale together and +share the card under load. On a zero-floor lane, it is the work and the tails a +cold warmth pays for each resident. A placed identity's demand is priced at its +chosen receipt, and a kept identity's at the slowest receipt it has for its +lane's cells, container shape and concurrency, under that lane's own ceiling. A +lane with fallback cells costs the hourly rate of its dearest cell, and a +candidate list's demand is priced at the slower of its two receipts. ### Envelope diff --git a/packages/sie_placement/src/sie_placement/planner.py b/packages/sie_placement/src/sie_placement/planner.py index 33df139e6..c6526e53a 100644 --- a/packages/sie_placement/src/sie_placement/planner.py +++ b/packages/sie_placement/src/sie_placement/planner.py @@ -522,13 +522,14 @@ def _boot_index(records: Iterable[BootEvidence], placements: Mapping[str, _Place return index -def _hourly_requests(demand: Demand) -> dict[str, tuple[int, ...]]: +def _hourly_requests(demand: Demand) -> dict[str, dict[datetime, int]]: hours: defaultdict[str, Counter[datetime]] = defaultdict(Counter) for series in demand.series: for hour in series.hours: hours[series.identity][hour.start] += hour.requests return { - identity: tuple(count for _, count in sorted(counts.items()) if count) for identity, counts in hours.items() + identity: {start: count for start, count in sorted(counts.items()) if count} + for identity, counts in hours.items() } @@ -579,7 +580,8 @@ def __init__( active_floor=parameters.active_floor, cooldown_s=parameters.cooldown_s, ) - self.requests = _hourly_requests(demand) + self.requests_by_hour = _hourly_requests(demand) + self.requests = {identity: tuple(counts.values()) for identity, counts in self.requests_by_hour.items()} self.current_plan = current_plan self.current: dict[str, tuple[PlanIdentity, PlanLane]] = {} self.current_lanes: dict[str, PlanLane] = {} @@ -1643,6 +1645,20 @@ def stay_beside(self, choices: dict[str, _Choice], originals: Mapping[str, _Choi choices, layout, total = trial, trial_layout, trial_total return choices + def lane_needs(self, rates: Mapping[str, float]) -> list[int]: + load: Counter[datetime] = Counter() + for identity, rate in rates.items(): + for start, count in self.requests_by_hour.get(identity, {}).items(): + load[start] += count / (rate * SECONDS_PER_HOUR) + return [math.ceil(round(containers, 9)) for containers in load.values()] + + def lane_rates(self, layout: _Layout, name: str) -> dict[str, float]: + return { + entry.identity: receipt.throughput.requests_per_second + for entry in layout.plan.identities + if entry.lane == name and (receipt := layout.receipts.get(entry.identity)) is not None + } + def riding_cost(self, lane: PlanLane, identity: str, receipt: Receipt) -> float: window = replace(self.window, ceiling=lane.max_containers) requests = self.requests.get(identity, ()) @@ -1830,6 +1846,10 @@ def floor_cost(identity: str) -> float: }, } trial_layout = self.assemble(trial) + name = trial_layout.names[identity] + ceiling = next(lane.max_containers for lane in trial_layout.plan.lanes if lane.name == name) + if any(need > ceiling for need in self.lane_needs(self.lane_rates(trial_layout, name))): + continue trial_total = self.total(trial_layout) saving = total - trial_total if round(saving, 2) <= 0: @@ -2066,6 +2086,16 @@ def _pricing( demand = {identity: self.demand_hours(lane, identity, receipts.get(identity)) for identity in hosted} priced = {identity: value for identity, value in demand.items() if value is not None} costs.update((identity, hourly * (floor + value)) for identity, value in priced.items()) + above = sum(priced.values()) + rates = { + identity: receipt.throughput.requests_per_second + for identity in priced + if (receipt := receipts.get(identity)) is not None + } + if lane.envelope.floor_min and len(rates) > 1: + above = self.window.scale * sum( + max(0, min(lane.max_containers, need) - lane.envelope.floor_min) for need in self.lane_needs(rates) + ) memory = _UNPACKABLE if len(hosted) > 1: memory = self.memory( @@ -2078,7 +2108,7 @@ def _pricing( name=lane.name, identities=hosted, hourly_cost=hourly, - monthly_cost=hourly * (floor + sum(priced.values())), + monthly_cost=hourly * (floor + above), floor_monthly_cost=hourly * lane.envelope.floor_min * hours, envelope_max_monthly_cost=hourly * lane.envelope.floor_max * hours, unpriced_identities=tuple(identity for identity in hosted if identity not in priced), diff --git a/packages/sie_placement/tests/test_packing.py b/packages/sie_placement/tests/test_packing.py index 2129301cd..e6c4f3084 100644 --- a/packages/sie_placement/tests/test_packing.py +++ b/packages/sie_placement/tests/test_packing.py @@ -408,6 +408,33 @@ def test_two_never_cold_identities_share_a_card_on_their_co_residency_receipt(): assert again.report.identity("org/a").chosen.co_residents == ("org/b",) +BUSY = {"org/a": {0: 20000}, "org/b": {0: 20000}} + + +def test_a_shared_lane_prices_its_residents_demand_together(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), hourly=BUSY) + assert placements(result) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + (lane,) = result.report.lanes + assert lane.monthly_cost == pytest.approx(MONTH + MONTH / 24) + assert result.report.monthly_cost == pytest.approx(MONTH + MONTH / 24) + + +def test_a_packing_whose_residents_together_need_more_containers_than_the_ceiling_is_not_made(): + result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(), hourly=BUSY, max_containers=1) + assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + assert result.report.monthly_cost == pytest.approx(2 * MONTH) + assert not any( + Measurement.CO_RESIDENCY_RECEIPT in request.measure for request in result.report.measurement_requests + ) + quiet = run( + levels(**{"org/a": 0, "org/b": 0}), + pair_receipts(), + hourly={"org/a": {0: 10000}, "org/b": {0: 10000}}, + max_containers=1, + ) + assert placements(quiet) == {"org/a": ("org-b-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)} + + def test_a_co_residency_receipt_over_a_warm_bound_keeps_the_identities_apart(): result = run(levels(**{"org/a": 0, "org/b": 0}), pair_receipts(b_p99=300.0)) assert placements(result) == {"org/a": ("org-a-l4", Warmth.PINNED), "org/b": ("org-b-l4", Warmth.PINNED)}