diff --git a/apis/inferenceclasses/definition.yaml b/apis/inferenceclasses/definition.yaml index 608dd2ad9..a84375d5b 100644 --- a/apis/inferenceclasses/definition.yaml +++ b/apis/inferenceclasses/definition.yaml @@ -611,6 +611,18 @@ spec: default: 1 minimum: 1 maximum: 64 + allowMultipleAllocations: + type: boolean + description: >- + Whether several claims can allocate one of these + devices at once, each consuming part of its capacity, + as DRA's consumable capacity (KEP-5075) allows. Set it + only when the DRA driver publishes the device with + allowMultipleAllocations. A ModelDeployment request + that sets capacity.requests matches only a device that + allows multiple allocations, and is scheduled by the + capacity its device has free. Only a claim: DRA device + can set it. Omitted, the scheduler counts the device whole. attributes: type: object description: >- @@ -655,6 +667,8 @@ spec: x-kubernetes-validations: - rule: "self.claim != 'DRA' || has(self.deviceClassName)" message: "deviceClassName is required when claim is DRA" + - rule: "self.claim == 'DRA' || !has(self.allowMultipleAllocations) || !self.allowMultipleAllocations" + message: "allowMultipleAllocations requires claim: DRA" status: type: object properties: diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 455678eb0..81c2976c3 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -870,6 +870,33 @@ spec: additionalProperties: type: integer minimum: 0 + sharedDevices: + type: object + description: >- + The free capacity of each of the nodes' devices + that allows multiple allocations and that a + claim has allocated, keyed by DeviceClass. Such a + device doesn't count in freeDevices. One no claim + has allocated does, with all of its capacity + free. + maxProperties: 32 + additionalProperties: + type: array + maxItems: 64 + x-kubernetes-list-type: atomic + items: + type: object + properties: + freeCapacity: + type: object + description: >- + The device's capacity no claim has + consumed, keyed by the names its driver + publishes capacity under. + maxProperties: 32 + additionalProperties: + type: string + maxLength: 32 nodes: type: integer minimum: 1 @@ -900,6 +927,8 @@ spec: maxLength: 253 count: type: integer + allowMultipleAllocations: + type: boolean attributes: type: object maxProperties: 32 diff --git a/apis/modeldeployments/definition.yaml b/apis/modeldeployments/definition.yaml index 7d8373264..0d2d8aaac 100644 --- a/apis/modeldeployments/definition.yaml +++ b/apis/modeldeployments/definition.yaml @@ -542,6 +542,43 @@ spec: device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("141Gi")) >= 0. minLength: 1 maxLength: 10240 + capacity: + type: object + description: >- + The share of each device's capacity the + request consumes, mirroring a DRA + DeviceRequest's capacity and carried + through to the ResourceClaim. Setting it + makes the request match only a device + whose InferenceClass sets + allowMultipleAllocations and declares at + least the requested amount of each + capacity. The scheduler then places the + request on devices with that much + capacity free, so several requests can + share one device. Omitted, the request + takes whole devices. + required: [requests] + properties: + requests: + type: object + description: >- + The capacity to consume on each + device, keyed by the names the + device declares capacity under (e.g. + memory); values are Kubernetes + Quantities. When exactly one of a + member's requests asks for memory, + its engine container gets + MODELPLANE_DEVICE_MEMORY_FRACTION: + the requested memory over the memory + the device's class declares. + minProperties: 1 + maxProperties: 8 + additionalProperties: + type: string + pattern: '^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$' + maxLength: 32 # This object has no schema default on purpose: the # apiserver applies defaults before CEL validation, so # defaulting it would inject it onto Standalone and diff --git a/apis/modelreplicas/definition.yaml b/apis/modelreplicas/definition.yaml index 6517f73a9..7ef5efd9a 100644 --- a/apis/modelreplicas/definition.yaml +++ b/apis/modelreplicas/definition.yaml @@ -376,6 +376,21 @@ spec: type: string minLength: 1 maxLength: 10240 + capacity: + type: object + description: >- + The capacity to consume on each device, + copied verbatim from the nodeSelector + request into the DeviceRequest. + required: [requests] + properties: + requests: + type: object + minProperties: 1 + maxProperties: 8 + additionalProperties: + type: string + maxLength: 32 worker: type: object # No schema default, mirroring the ModelDeployment diff --git a/design/gpu-sharing.md b/design/gpu-sharing.md new file mode 100644 index 000000000..77fbee411 --- /dev/null +++ b/design/gpu-sharing.md @@ -0,0 +1,135 @@ +# Sharing a GPU between replicas + +**Status:** Draft +**Date:** November 2026 +**Author:** Tobias Kässer (@kaessert) + +This document proposes that the fleet scheduler place replicas by the capacity +a GPU has free, where the GPU's DRA driver lets several claims share it, so two +small models can run on one GPU +([#644](https://github.com/modelplaneai/modelplane/issues/644)). It builds on +[free-gpu-scheduling.md](./free-gpu-scheduling.md). + +## Background + +DRA's consumable capacity +([KEP-5075](https://github.com/kubernetes/enhancements/issues/5075)) lets a +driver publish a device with `allowMultipleAllocations`, and a claim request a +share of its capacity with `capacity.requests`. The scheduler records what each +claim consumed in `status.allocation.devices.results[].consumedCapacity`, and +allocates the device to further claims until their shares add up to its +capacity. It's alpha behind the `DRAConsumableCapacity` feature gate in +Kubernetes 1.34 and 1.35, and on by default from 1.36. The NVIDIA DRA driver +publishes its GPUs this way with its `ConsumableShares` feature gate and a +`consumableShares` mode set. In its +`unlimited` mode a claim that requests no memory consumes none, and in its +`memory` mode it consumes the whole GPU. + +On a k3s 1.35 cluster with the gate on, and NVIDIA's DRA driver v0.5.0 in +`unlimited` mode, a ModelDeployment's claim and a plain pod's claim were both +allocated the same GPU, each with a share of its own. A second ModelDeployment didn't schedule, reporting +`InsufficientCapacity`. The cluster agent counts a device as taken once any +claim has allocated it, so the pool's `nodesByFreeDevices` showed no GPU free. +A ModelDeployment also has no way to ask for a share, and an engine no way to +learn how much of the GPU's memory it may use. + +## Proposal + +A class marks the devices that several claims may share, and a deployment's +request states how much of each device it consumes, both in DRA's terms: + +```yaml +# The InferenceClass. +spec: + devices: + - name: gpu + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 1 + allowMultipleAllocations: true # New. + capacity: + memory: { value: "81559Mi" } +--- +# A ModelDeployment member. +nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: device.driver == "gpu.nvidia.com" + capacity: # New. + requests: + memory: 24Gi +``` + +A request that sets `capacity.requests` matches only a device that allows +multiple allocations and declares at least the requested amount of each +capacity. A deployment doesn't name a class, so CEL on its XRD can't check +this, and the check is part of matching. + +The cluster agent reports, in each bucket of a pool's histogram, the capacity +each shared device has left. A shared device is one that allows multiple +allocations and that a claim has allocated. Its free capacity is its capacity +less the claims' `consumedCapacity`, and none of it if a claim has no recorded +consumption. It still doesn't count in `freeDevices`, so `freeDevices` keeps +meaning devices no claim holds: + +```yaml +nodesByFreeDevices: +- freeDevices: + gpu.nvidia.com: 0 + sharedDevices: # New. + gpu.nvidia.com: + - freeCapacity: + memory: 56Gi + nodes: 1 +``` + +The scheduler fits a request that sets `capacity.requests` on `count` distinct +devices, each a shared device with that much free or a device in +`freeDevices`, which has the capacity its class declares. It takes shared +devices first, the fullest first, guessing that DRA packs a device before +starting another. A request that sets no `capacity.requests` takes whole +devices from `freeDevices`, as it does on any pool. That's stricter than DRA in +NVIDIA's `unlimited` mode, where such a claim consumes nothing, and the same as +DRA without a request policy or in `memory` mode. The scheduler can't see which +mode a driver runs in, and the stricter guess can't overcommit a device. A pool +the agent doesn't report on is still counted in whole nodes. + +compose-model-deployment copies `capacity.requests` onto the ModelReplica's +device requests, and compose-model-replica into the `ResourceClaimTemplate`. So +each pod gets a claim of its own, and DRA places and accounts for each share. +When exactly one of a member's requests asks for memory, its engine container +gets `MODELPLANE_DEVICE_MEMORY_FRACTION`, the requested memory over the memory +the class declares, rounded down to three places. DRA accounts for a share but +doesn't confine an engine to it, so Modelplane tells the engine. Modelplane +doesn't write engine flags, so the variable is for the deployment's args to +reference, as in `--gpu-memory-utilization=$(MODELPLANE_DEVICE_MEMORY_FRACTION)`. + +Turning sharing on stays with the platform team: the feature gate, and the +driver's sharing mode. + +## Future improvements + +The serving stack could configure the NVIDIA DRA driver's sharing mode from the +InferenceCluster on clusters Modelplane provisions. + +Per-replica GPU metrics would need the share each replica's pods hold, which +the ReplicaObservation could report from the same `consumedCapacity`. + +## Alternatives considered + +### Share one ResourceClaim per pool + +Every replica sharing a GPU could reference one ResourceClaim, which works +without consumable capacity. DRA would then account for nothing: each replica's +share would be Modelplane's bookkeeping alone, and every pod on the claim would +see the same devices and have to run on their node. The +`capacity.requests` field doesn't preclude adding it later for clusters before +1.36. + +### Write the share into known engines' args + +compose-model-replica could add `--gpu-memory-utilization` for vLLM and its +equivalent for other engines. Modelplane passes engine args through as written +and doesn't know any engine by name, and a variable keeps it that way. diff --git a/docs/content/architecture/scheduling.md b/docs/content/architecture/scheduling.md index 1ecc402b1..1183f13bd 100644 --- a/docs/content/architecture/scheduling.md +++ b/docs/content/architecture/scheduling.md @@ -129,6 +129,27 @@ NUMA node, with that node's cores as its capacity, and the agent counts a device as taken once any pod claims some of its cores. A pool's nodes fit at most one CPU replica per NUMA node, whatever number of cores each claims. +A device whose class sets `allowMultipleAllocations` can be shared. Once a pod +claims part of one, the agent stops counting it in `freeDevices` and reports +the capacity it has left instead, under `sharedDevices`: + +```yaml + nodesByFreeDevices: + - freeDevices: + gpu.nvidia.com: 0 + sharedDevices: + gpu.nvidia.com: + - freeCapacity: + memory: 56Gi + nodes: 1 +``` + +A pod whose request sets `capacity.requests` fits a device with that much free, +either a shared one or one in `freeDevices`, which has all the capacity its +class declares. One that sets none takes whole devices from `freeDevices`, as on +any other pool. That's stricter than DRA when the driver consumes nothing for a +claim that requests nothing, and the same when it consumes the whole device. + The scheduler counts a pool in whole nodes when the agent hasn't reported on it or can't observe it. Such a pool publishes no `observedNodes`, and every node it can have starts empty. A pod that claims any device takes a whole one. diff --git a/docs/content/models/model-deployment.md b/docs/content/models/model-deployment.md index 927a14903..d4662958f 100644 --- a/docs/content/models/model-deployment.md +++ b/docs/content/models/model-deployment.md @@ -168,6 +168,34 @@ nodeSelector: - cel: device.attributes["nic.nvidia.com"].linkType == "infiniband" ``` +### Sharing a GPU + +When the platform team's class allows a GPU to be shared, several pods can each +take part of its memory instead of a whole GPU. Request the share with +`capacity.requests`: + +```yaml {nocopy=true} +nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: device.driver == "gpu.nvidia.com" + capacity: + requests: + memory: 24Gi +``` + +The request matches only a device whose class sets `allowMultipleAllocations` +and declares at least that much memory, and the scheduler places it on a GPU +with that much memory free. Modelplane copies `capacity.requests` into the pod's +`ResourceClaim`, and sets `MODELPLANE_DEVICE_MEMORY_FRACTION` in the engine +container to the requested memory over the device's, `0.3` for 24Gi of an 80Gi +GPU. DRA accounts for the share but doesn't confine the engine to it, so pass it +to the engine, such as with vLLM's +`--gpu-memory-utilization=$(MODELPLANE_DEVICE_MEMORY_FRACTION)`. A request without +`capacity` takes whole GPUs, even on a class that allows sharing. + ### What you can match on Each selector is evaluated against one device and must return a boolean. The diff --git a/docs/content/platform/inference-class.md b/docs/content/platform/inference-class.md index 7a215173b..b59cadfef 100644 --- a/docs/content/platform/inference-class.md +++ b/docs/content/platform/inference-class.md @@ -46,6 +46,33 @@ Each device sets a `claim` discriminator: hardware that matters for placement but has no DRA driver yet, like an InfiniBand fabric. +## Shared devices + +Set `allowMultipleAllocations: true` on a `DRA` device whose driver publishes it +that way. Several pods can then share the device, each consuming part of its +capacity. This is Kubernetes' consumable capacity +([KEP-5075](https://github.com/kubernetes/enhancements/issues/5075)). On +Kubernetes 1.34 and 1.35 it needs the `DRAConsumableCapacity` feature gate on the +API server, controller manager, scheduler, and Kubelet. From 1.36 it's on by +default. The NVIDIA DRA driver publishes GPUs this way with its +`ConsumableShares` feature gate and a `consumableShares` mode set. A claim that +doesn't request memory consumes none of a GPU in the driver's `unlimited` mode, +and the whole GPU in its `memory` mode. + +```yaml {nocopy=true} +devices: +- name: gpu + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 1 + allowMultipleAllocations: true + capacity: + memory: { value: "81559Mi" } +``` + +A `ModelDeployment` takes a share of such a device with `capacity.requests`, and +the scheduler places it by the capacity each device has free. + ## The device contract The `driver`, attribute keys, and capacity keys a class declares are a contract diff --git a/functions/compose-model-deployment/function/fn.py b/functions/compose-model-deployment/function/fn.py index b2e0c57ef..51b756c49 100644 --- a/functions/compose-model-deployment/function/fn.py +++ b/functions/compose-model-deployment/function/fn.py @@ -92,6 +92,33 @@ # ModelService rather than a deployment, so something has to reconcile the two. SERVED_MODEL_NAME_ENV = "MODELPLANE_SERVED_MODEL_NAME" +# Injected into the engine container of a member that requests a share of its +# device's memory, so the engine can size itself to that share: +# +# args: +# - --gpu-memory-utilization=$(MODELPLANE_DEVICE_MEMORY_FRACTION) +# +# DRA accounts a shared device's memory but doesn't enforce it, so an engine +# that takes all of a device's memory starves the others sharing it. +DEVICE_MEMORY_FRACTION_ENV = "MODELPLANE_DEVICE_MEMORY_FRACTION" + + +def _inject_device_memory_fraction(template: mrv1alpha1.Template, requests: list[scheduling.DeviceRequest]) -> None: + """Put DEVICE_MEMORY_FRACTION_ENV ahead of the engine container's own env entries. + + Only when exactly one of the member's device requests asks for memory: one + variable can't describe two devices. A user entry of the same name is + dropped, as _inject_served_model_name drops one. + """ + fractions = [r.memory_fraction for r in requests if r.memory_fraction] + if len(fractions) != 1: + return + for container in template.spec.containers: + if container.name != "engine": + continue + existing = [e for e in container.env or [] if e.name != DEVICE_MEMORY_FRACTION_ENV] + container.env = [mrv1alpha1.EnvItem(name=DEVICE_MEMORY_FRACTION_ENV, value=fractions[0]), *existing] + def _inject_served_model_name(template: mrv1alpha1.Template, served: str) -> None: """Put SERVED_MODEL_NAME_ENV ahead of the engine container's own env entries. @@ -499,6 +526,7 @@ def _replica_engine(self, engine: v1alpha1.Engine, placement: scheduling.EngineP members = [] for member, mp in zip(engine.members, placement.members, strict=True): template = mrv1alpha1.Template.model_validate(member.template.model_dump(exclude_unset=True)) + _inject_device_memory_fraction(template, mp.device_requests) _inject_served_model_name(template, served) replica_member = mrv1alpha1.Member( role=member.role, @@ -518,6 +546,11 @@ def _replica_engine(self, engine: v1alpha1.Engine, placement: scheduling.EngineP ) for r in mp.device_requests ] + for r, dr in zip(mp.device_requests, replica_member.deviceRequests, strict=True): + # Set only when requested, so a request that takes whole + # devices doesn't serialize an empty capacity. + if r.capacity_requests: + dr.capacity = mrv1alpha1.Capacity(requests=dict(r.capacity_requests)) # Only a Worker carries the worker block, and only when the user # set one. Setting worker=None explicitly would serialize a literal # null into the composed manifest rather than omitting the field. diff --git a/functions/compose-model-deployment/function/scheduling.py b/functions/compose-model-deployment/function/scheduling.py index 6af7d4af0..b7d376461 100644 --- a/functions/compose-model-deployment/function/scheduling.py +++ b/functions/compose-model-deployment/function/scheduling.py @@ -77,12 +77,22 @@ report on a pool, every one of the nodes it can have starts with every device free, and a pod that claims any device takes a whole node. +A device whose class allows multiple allocations can be shared. A request that +sets capacity.requests consumes that much of each device it's placed on, and +fits a device with that much capacity free: one no claim has allocated, which +has all the capacity its class declares, or one the agent reports shared with +enough left. A request without capacity.requests takes whole devices no claim +has allocated, as it does on any other pool. That's stricter than DRA when the +driver's request policy consumes nothing by default, and matches it when the +policy, or its absence, consumes the whole device. + The scheduler predicts and the workload cluster's scheduler decides. It can't know which node each pod will bind to, so it guesses, and a wrong guess leaves a pod Pending until devices free up. The next reconcile sees the updated observed state. """ +import decimal from dataclasses import dataclass, field from models.ai.modelplane.inferencecluster import v1alpha1 as icv1alpha1 @@ -90,7 +100,7 @@ from models.ai.modelplane.modelreplica import v1alpha1 as mrv1alpha1 from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 -from function import cel +from function import cel, quantity def _name(meta: metav1.ObjectMeta | None) -> str: @@ -149,21 +159,79 @@ def _published_count(value: int | None) -> int: return value +# A device's capacity, or the capacity a request consumes on one, as sorted +# (name, amount) pairs, so devices with the same capacity compare equal. +_Capacity = tuple[tuple[str, decimal.Decimal], ...] + +# The capacity name whose share compose-model-deployment tells the engine. +_CAPACITY_MEMORY = "memory" + + +def _parse_capacity(amounts: dict[str, str]) -> _Capacity | None: + """Parse capacity quantities keyed by name, or return None if one doesn't parse.""" + try: + return tuple(sorted((name, quantity.parse(amount)) for name, amount in amounts.items())) + except ValueError: + return None + + +def _covers(capacity: _Capacity, wants: _Capacity) -> bool: + """Whether a device's capacity has at least the amount of each capacity wanted.""" + have = dict(capacity) + return all(name in have and have[name] >= amount for name, amount in wants) + + +def _consume(capacity: _Capacity, wants: _Capacity) -> _Capacity: + """A device's capacity less what a request consumes of it.""" + left = dict(capacity) + for name, amount in wants: + left[name] -= amount + return tuple(sorted(left.items())) + + +def _device_capacity(device: icv1alpha1.Device) -> _Capacity: + """The capacity a pool device declares. A value that doesn't parse is left out, so nothing can consume it.""" + capacity = [] + for name, c in (device.capacity or {}).items(): + try: + capacity.append((name, quantity.parse(c.value))) + except ValueError: + continue + return tuple(sorted(capacity)) + + +def _memory_fraction(wants: _Capacity, device: icv1alpha1.Device) -> str: + """The requested memory over the device's, rounded down to three places, or empty if either is unknown.""" + requested = dict(wants).get(_CAPACITY_MEMORY) + total = dict(_device_capacity(device)).get(_CAPACITY_MEMORY) + if requested is None or not total: + return "" + fraction = (requested / total).quantize(decimal.Decimal("0.001"), rounding=decimal.ROUND_DOWN) + return f"{fraction.normalize():f}" + + @dataclass class DeviceRequest: """A resolved DRA device request for a matched pool device. Carries everything compose-model-replica needs to emit one DeviceRequest in a ResourceClaim: the request name, the DeviceClass to claim through (from the - matched InferenceClass device), the count, and the CEL selectors. Only - claim: DRA devices produce one of these; synthetic devices are matched for - scheduling but never claimed. + matched InferenceClass device), the count, the CEL selectors, and the + capacity it consumes on each device of a class that allows multiple + allocations. Only claim: DRA devices produce one of these; synthetic devices + are matched for scheduling but never claimed. + + memory_fraction is the requested memory over the memory the matched device + declares, as a decimal string rounded down to three places. It's empty + unless the request asks for memory. """ name: str device_class_name: str count: int cel_selectors: list[str] + capacity_requests: dict[str, str] = field(default_factory=dict) + memory_fraction: str = "" @dataclass @@ -231,13 +299,17 @@ class _CompiledRequest: """One nodeSelector device request with its CEL selectors compiled. cel_selectors are the raw expressions (carried through to the DeviceRequest); - programs are the compiled forms used to match a pool device. + programs are the compiled forms used to match a pool device. capacity_requests + are the raw capacity.requests (carried through too); capacity is the same + parsed, or None when one doesn't parse, which matches no device. """ name: str count: int cel_selectors: list[str] programs: list[cel.Program] + capacity_requests: dict[str, str] = field(default_factory=dict) + capacity: _Capacity | None = () @dataclass @@ -291,12 +363,15 @@ def _compile_requests(node_selector: mdv1alpha1.NodeSelector) -> list[_CompiledR requests = [] for req in node_selector.devices: cel_selectors = [s.cel for s in req.selectors if s.cel] + capacity_requests = dict(req.capacity.requests) if req.capacity else {} requests.append( _CompiledRequest( name=req.name, count=int(req.count or 1), cel_selectors=cel_selectors, programs=[cel.Program(c) for c in cel_selectors], + capacity_requests=capacity_requests, + capacity=_parse_capacity(capacity_requests), ) ) return requests @@ -415,6 +490,11 @@ def _match_member(pool: icv1alpha1.SchedulablePool, member: _CompiledMember) -> false-reject, never overcommit, so it fails safe. Members are matched independently of each other, against what one node of the pool has. Whether their pods fit the nodes the pool has free is up to its capacity. + + A request that sets capacity.requests matches only a device that allows + multiple allocations and declares at least the requested amount of each + capacity, as DRA's consumable capacity requires. It still needs count of + them: it consumes its share of count distinct devices. """ devices = pool.devices or [] remaining = [_published_count(d.count) for d in devices] @@ -426,6 +506,8 @@ def _match_member(pool: icv1alpha1.SchedulablePool, member: _CompiledMember) -> continue if not _device_satisfies(device, req.programs): continue + if req.capacity_requests and not _shareable(device, req.capacity): + continue match = device remaining[i] -= req.count break @@ -438,17 +520,40 @@ def _match_member(pool: icv1alpha1.SchedulablePool, member: _CompiledMember) -> device_class_name=match.deviceClassName or "", count=req.count, cel_selectors=req.cel_selectors, + capacity_requests=req.capacity_requests, + memory_fraction=_memory_fraction(req.capacity or (), match), ) ) return resolved -# The devices one pod claims, keyed by DeviceClass. -_Claim = dict[str, int] +def _shareable(device: icv1alpha1.Device, wants: _Capacity | None) -> bool: + """Whether a pool device can give a request the capacity it wants while other claims hold it too.""" + if wants is None or not device.allowMultipleAllocations: + return False + return (device.claim or _CLAIM_DRA) == _CLAIM_DRA and _covers(_device_capacity(device), wants) + + +@dataclass(frozen=True) +class _Take: + """The devices one of a pod's requests takes: count devices of a DeviceClass. + + capacity is what the request consumes on each of them when they're shared. + It's empty for a request that takes whole devices. + """ + + device_class: str + count: int + capacity: _Capacity = () + -# A node's free devices as sorted (DeviceClass, count) pairs, so nodes with the -# same free devices share a key. -_Free = tuple[tuple[str, int], ...] +# The devices one pod claims, one _Take per device request. +_Claim = tuple[_Take, ...] + + +def _size(claim: _Claim) -> int: + """How many devices a pod claims.""" + return sum(t.count for t in claim) def _claim(requests: list[DeviceRequest]) -> _Claim: @@ -459,10 +564,10 @@ def _claim(requests: list[DeviceRequest]) -> _Claim: still use CPU and memory on whatever node of its gang's pool the cluster's scheduler packs them onto, which the scheduler doesn't model. """ - claim: _Claim = {} - for r in requests: - claim[r.device_class_name] = claim.get(r.device_class_name, 0) + r.count - return claim + return tuple( + _Take(device_class=r.device_class_name, count=r.count, capacity=_parse_capacity(r.capacity_requests) or ()) + for r in requests + ) def _observed_claim(member: mrv1alpha1.Member) -> _Claim: @@ -472,10 +577,14 @@ def _observed_claim(member: mrv1alpha1.Member) -> _Claim: now. For a retained member whose pinned pool drifted, the two can disagree for one reconcile, until the re-resolved spec is observed. """ - claim: _Claim = {} - for r in member.deviceRequests or []: - claim[r.deviceClassName] = claim.get(r.deviceClassName, 0) + int(r.count or 1) - return claim + return tuple( + _Take( + device_class=r.deviceClassName, + count=int(r.count or 1), + capacity=(_parse_capacity(dict(r.capacity.requests)) or ()) if r.capacity else (), + ) + for r in member.deviceRequests or [] + ) def _schedulable_pools(cluster: icv1alpha1.InferenceCluster) -> list[icv1alpha1.SchedulablePool]: @@ -514,6 +623,58 @@ def _replica_index(replica: mrv1alpha1.ModelReplica) -> int: return 0 +@dataclass(frozen=True, order=True) +class _Node: + """A node's free devices, so nodes with the same free devices share a key. + + free is how many devices of each DeviceClass no claim has allocated, as + sorted (DeviceClass, count) pairs. shared is the free capacity of each + device of a class that allows multiple allocations that a claim has + allocated, as sorted (DeviceClass, capacities) pairs, each capacities sorted + too. A device no claim has allocated counts in free, with all the capacity + its class declares. + """ + + free: tuple[tuple[str, int], ...] + shared: tuple[tuple[str, tuple[_Capacity, ...]], ...] = () + + def take(self, claim: _Claim, whole: dict[str, _Capacity]) -> "_Node | None": + """The node with a pod's claim placed on it, or None if the claim doesn't fit. + + A take with no capacity takes count devices no claim has allocated. One + with capacity consumes it on count distinct devices that have that much + free. It uses shared devices before free ones, the fullest that fits + first, guessing that DRA packs a device before it starts another; whole + gives a free device's capacity, by DeviceClass. Takes are placed in + order, so two of a pod's requests may share one device, as DRA allows + two requests of a claim to. + """ + free = dict(self.free) + shared = {c: list(devices) for c, devices in self.shared} + for t in claim: + if not t.capacity: + if free.get(t.device_class, 0) < t.count: + return None + free[t.device_class] -= t.count + continue + devices = shared.setdefault(t.device_class, []) + fits = sorted((d, i) for i, d in enumerate(devices) if _covers(d, t.capacity))[: t.count] + for d, i in fits: + devices[i] = _consume(d, t.capacity) + rest = t.count - len(fits) + if rest == 0: + continue + capacity = whole.get(t.device_class) + if capacity is None or not _covers(capacity, t.capacity) or free.get(t.device_class, 0) < rest: + return None + free[t.device_class] -= rest + devices.extend([_consume(capacity, t.capacity)] * rest) + return _Node( + free=tuple(sorted(free.items())), + shared=tuple(sorted((c, tuple(sorted(devices))) for c, devices in shared.items() if devices)), + ) + + @dataclass class _Pool: """A pool's free capacity: how many of its nodes have each set of free devices. @@ -521,14 +682,18 @@ class _Pool: whole_node is set for a pool the cluster agent hasn't reported on. Each of its nodes starts with every device free, and a pod that claims any device takes a whole node, however few of its devices it claims. + + whole is the capacity of a device no claim has allocated, for each + DeviceClass whose devices allow multiple allocations. """ - nodes: dict[_Free, int] + nodes: dict[_Node, int] whole_node: bool + whole: dict[str, _Capacity] = field(default_factory=dict) def copy(self) -> "_Pool": """A copy to place pods on without changing this one.""" - return _Pool(nodes=dict(self.nodes), whole_node=self.whole_node) + return _Pool(nodes=dict(self.nodes), whole_node=self.whole_node, whole=self.whole) def place(self, pods: list[_Claim]) -> bool: """Place one copy of an engine's pods, each claiming its devices, or return False if they don't all fit. @@ -545,23 +710,23 @@ def place(self, pods: list[_Claim]) -> bool: stops the cluster's scheduler binding them to one. """ fit = True - for claim in sorted(pods, key=lambda c: sum(c.values())): + for claim in sorted(pods, key=_size): if not claim: continue - fits = [f for f in self.nodes if self.whole_node or all(dict(f).get(c, 0) >= n for c, n in claim.items())] + placed = {n: n if self.whole_node else n.take(claim, self.whole) for n in self.nodes} + fits = [n for n, left in placed.items() if left is not None] if not fits: fit = False continue - node = max(fits, key=lambda f: (sum(dict(f).get(c, 0) for c in claim), f)) + classes = {t.device_class for t in claim} + node = max(fits, key=lambda n: (sum(dict(n.free).get(c, 0) for c in classes), n)) self.nodes[node] -= 1 if self.nodes[node] == 0: del self.nodes[node] if self.whole_node: continue - free = dict(node) - for c, n in claim.items(): - free[c] -= n - left = tuple(sorted(free.items())) + left = placed[node] + assert left is not None self.nodes[left] = self.nodes.get(left, 0) + 1 return fit @@ -576,26 +741,47 @@ def _pool_capacity(pool: icv1alpha1.SchedulablePool) -> _Pool: """ # Keyed as _match_member keys the requests it resolves. full: dict[str, int] = {} + whole: dict[str, _Capacity] = {} for d in pool.devices or []: if (d.claim or _CLAIM_DRA) == _CLAIM_DRA: key = d.deviceClassName or "" full[key] = full.get(key, 0) + _published_count(d.count) - empty = tuple(sorted(full.items())) + if d.allowMultipleAllocations: + whole.setdefault(key, _device_capacity(d)) + empty = _Node(free=tuple(sorted(full.items()))) nodes = _published_count(pool.nodes) if pool.observedNodes is None: - return _Pool(nodes={empty: nodes} if nodes else {}, whole_node=True) + return _Pool(nodes={empty: nodes} if nodes else {}, whole_node=True, whole=whole) - capacity = _Pool(nodes={}, whole_node=False) + capacity = _Pool(nodes={}, whole_node=False, whole=whole) for bucket in pool.nodesByFreeDevices or []: - free = tuple(sorted((bucket.freeDevices or {}).items())) - capacity.nodes[free] = capacity.nodes.get(free, 0) + bucket.nodes + node = _Node(free=tuple(sorted((bucket.freeDevices or {}).items())), shared=_shared(bucket)) + capacity.nodes[node] = capacity.nodes.get(node, 0) + bucket.nodes growth = nodes - pool.observedNodes if growth > 0: capacity.nodes[empty] = capacity.nodes.get(empty, 0) + growth return capacity +def _shared(bucket: icv1alpha1.NodesByFreeDevice) -> tuple[tuple[str, tuple[_Capacity, ...]], ...]: + """The free capacity of a bucket's shared devices, as _Node keeps it. + + A device whose capacity doesn't parse is left out, as though it had none + free. + """ + shared = [] + for device_class, devices in (bucket.sharedDevices or {}).items(): + capacities = [] + for d in devices: + capacity = _parse_capacity(dict(d.freeCapacity or {})) + if capacity is not None: + capacities.append(capacity) + if capacities: + shared.append((device_class, tuple(sorted(capacities)))) + return tuple(sorted(shared)) + + @dataclass class _Ledger: """Free capacity per (cluster, pool). @@ -679,7 +865,7 @@ def _build_ledger( # Smallest pod first, as _place_engines places them. engines = sorted( r.spec.engines, - key=lambda e: max((sum(_observed_claim(m).values()) for m in e.members or []), default=0), + key=lambda e: max((_size(_observed_claim(m)) for m in e.members or []), default=0), ) for engine in engines: # The pods of one copy of the engine, by the pool they're pinned to. diff --git a/functions/compose-model-deployment/tests/test_fn.py b/functions/compose-model-deployment/tests/test_fn.py index c0e77f000..bdb8d69b5 100644 --- a/functions/compose-model-deployment/tests/test_fn.py +++ b/functions/compose-model-deployment/tests/test_fn.py @@ -22,7 +22,7 @@ import pytest from crossplane.function import resource from crossplane.function.proto.v1 import run_function_pb2 as fnv1 -from function import fn +from function import fn, scheduling from google.protobuf import duration_pb2 as durationpb from google.protobuf import json_format, message from google.protobuf import struct_pb2 as structpb @@ -65,6 +65,17 @@ class InjectNameCase: want: mrv1alpha1.Template +@dataclasses.dataclass +class InjectFractionCase: + """A test case for fn._inject_device_memory_fraction.""" + + name: str + reason: str + template: mrv1alpha1.Template + requests: list[scheduling.DeviceRequest] + want: mrv1alpha1.Template + + @dataclasses.dataclass class ServedModelNameCase: """A test case for fn.served_model_name.""" @@ -2726,6 +2737,221 @@ def _to_dict(msg: message.Message) -> dict: ], ), ), + # The deployment and cluster here differ from the helpers' in the GPU + # request and the pool's GPU, which allows multiple allocations, so they're + # written out. + ComposeCase( + name="SharedGPU", + reason="A replica requesting a share of a GPU's memory carries the request into its device requests and the share into its engine's env.", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + v1alpha1.ModelDeployment( + metadata=metav1.ObjectMeta(name="my-model", namespace="ml-team"), + spec=v1alpha1.SpecModel2( + replicas=1, + template=v1alpha1.TemplateModel1( + spec=v1alpha1.SpecModel1( + engines=[ + v1alpha1.Engine( + name="main", + members=[ + v1alpha1.Member( + role="Standalone", + nodeSelector=v1alpha1.NodeSelector( + devices=[ + v1alpha1.Device( + name="gpu", + count=1, + selectors=[ + v1alpha1.Selector( + cel='device.driver == "gpu.nvidia.com"' + ) + ], + capacity=v1alpha1.Capacity( + requests={"memory": "24Gi"} + ), + ), + ], + ), + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[ + v1alpha1.Container( + name="engine", image="vllm/vllm-openai:latest" + ) + ], + ), + ), + ) + ], + ) + ], + ), + ), + ), + ).model_dump(exclude_none=True, mode="json", by_alias=True) + ) + ), + ), + required_resources={ + "clusters": fnv1.Resources( + items=[ + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceCluster", + "metadata": {"name": "cluster-a"}, + "spec": { + "cluster": { + "source": "Existing", + "existing": {"secretRef": {"name": "k", "key": "kubeconfig"}}, + }, + "stack": "Standard", + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2025-01-01T00:00:00Z", + } + ], + "providerConfigRef": {"name": "cluster-a"}, + "gateway": {"address": "10.0.0.1", "hostname": "cluster.clusters.example.com"}, + "schedulablePools": [ + { + "name": "default", + "nodes": 1, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.nvidia.com", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "allowMultipleAllocations": True, + "capacity": {"memory": {"value": "80Gi"}}, + } + ], + } + ], + }, + } + ) + ) + ] + ), + "all-replicas": fnv1.Resources(), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=_desired_model_deployment(total_replicas=1, ready_replicas=0, ready=fnv1.READY_FALSE), + resources={ + "replica-cluster-a-0": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "ModelReplica", + "metadata": { + "name": "my-model-5ab63", + "namespace": "ml-team", + "labels": { + "modelplane.ai/deployment": "my-model", + "modelplane.ai/cluster": "cluster-a", + "modelplane.ai/replica-index": "0", + }, + }, + "spec": { + "clusterName": "cluster-a", + "engines": [ + { + "name": "main", + "copies": 1, + "members": [ + { + "role": "Standalone", + "nodePoolName": "default", + "deviceRequests": [ + { + "name": "gpu", + "deviceClassName": "gpu.nvidia.com", + "count": 1, + "selectors": [{"cel": 'device.driver == "gpu.nvidia.com"'}], + "capacity": {"requests": {"memory": "24Gi"}}, + } + ], + "template": { + "spec": { + "containers": [ + { + "name": "engine", + "image": "vllm/vllm-openai:latest", + # 24Gi of the GPU's 80Gi. + "env": [ + { + "name": "MODELPLANE_SERVED_MODEL_NAME", + "value": "ml-team/my-model", + }, + { + "name": "MODELPLANE_DEVICE_MEMORY_FRACTION", + "value": "0.3", + }, + ], + } + ] + } + }, + } + ], + } + ], + }, + } + ), + ), + }, + ), + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Scheduled 1 replicas across 1 clusters: cluster-a", + ), + ], + context=structpb.Struct(), + requirements=fnv1.Requirements( + resources={ + "clusters": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="InferenceCluster"), + "all-replicas": fnv1.ResourceSelector(api_version="modelplane.ai/v1alpha1", kind="ModelReplica"), + }, + ), + conditions=[ + fnv1.Condition( + type="ReplicasScheduled", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Scheduling", + message="Created 0 of 1 replicas", + ), + fnv1.Condition( + type="ReplicasServing", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas serving", + ), + fnv1.Condition( + type="ReplicasReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="ModelStarting", + message="0 of 1 replicas ready", + ), + ], + ), + ), ] @@ -2863,6 +3089,140 @@ def test_inject_name(case: InjectNameCase) -> None: assert got.model_dump() == case.want.model_dump(), case.reason +INJECT_FRACTION_CASES = [ + InjectFractionCase( + name="MemoryShare", + reason="A member requesting a share of one GPU's memory gets the share ahead of the env the user wrote.", + template=mrv1alpha1.Template( + spec=mrv1alpha1.Spec( + containers=[ + mrv1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + env=[mrv1alpha1.EnvItem(name="HF_TOKEN", value="x")], + ) + ] + ) + ), + requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + want=mrv1alpha1.Template( + spec=mrv1alpha1.Spec( + containers=[ + mrv1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + env=[ + mrv1alpha1.EnvItem(name="MODELPLANE_DEVICE_MEMORY_FRACTION", value="0.3"), + mrv1alpha1.EnvItem(name="HF_TOKEN", value="x"), + ], + ) + ] + ) + ), + ), + InjectFractionCase( + name="WholeDevice", + reason="A member that takes whole devices gets no memory fraction.", + template=mrv1alpha1.Template( + spec=mrv1alpha1.Spec(containers=[mrv1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]) + ), + requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[], + ) + ], + want=mrv1alpha1.Template( + spec=mrv1alpha1.Spec(containers=[mrv1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]) + ), + ), + # One variable can't say which of two devices a share is of. + InjectFractionCase( + name="TwoMemoryShares", + reason="A member requesting memory on two device requests gets no memory fraction.", + template=mrv1alpha1.Template( + spec=mrv1alpha1.Spec(containers=[mrv1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]) + ), + requests=[ + scheduling.DeviceRequest( + name="prefill", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ), + scheduling.DeviceRequest( + name="decode", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[], + capacity_requests={"memory": "40Gi"}, + memory_fraction="0.5", + ), + ], + want=mrv1alpha1.Template( + spec=mrv1alpha1.Spec(containers=[mrv1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")]) + ), + ), + InjectFractionCase( + name="UserOverride", + reason="A MODELPLANE_DEVICE_MEMORY_FRACTION the user wrote is replaced by the member's share.", + template=mrv1alpha1.Template( + spec=mrv1alpha1.Spec( + containers=[ + mrv1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + env=[mrv1alpha1.EnvItem(name="MODELPLANE_DEVICE_MEMORY_FRACTION", value="0.9")], + ) + ] + ) + ), + requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + want=mrv1alpha1.Template( + spec=mrv1alpha1.Spec( + containers=[ + mrv1alpha1.Container( + name="engine", + image="vllm/vllm-openai:latest", + env=[mrv1alpha1.EnvItem(name="MODELPLANE_DEVICE_MEMORY_FRACTION", value="0.3")], + ) + ] + ) + ), + ), +] + + +@pytest.mark.parametrize("case", INJECT_FRACTION_CASES, ids=lambda case: case.name) +def test_inject_fraction(case: InjectFractionCase) -> None: + """_inject_device_memory_fraction tells the engine its share of its device's memory.""" + got = case.template.model_copy(deep=True) # noqa: MPT401 # _inject_device_memory_fraction edits it in place. + fn._inject_device_memory_fraction(got, case.requests) + assert got.model_dump() == case.want.model_dump(), case.reason + + SERVED_MODEL_NAME_CASES = [ # So two deployments in different namespaces can't collide, and a # ModelService can rewrite one name for a whole deployment. diff --git a/functions/compose-model-deployment/tests/test_scheduling.py b/functions/compose-model-deployment/tests/test_scheduling.py index cdcc90785..965439958 100644 --- a/functions/compose-model-deployment/tests/test_scheduling.py +++ b/functions/compose-model-deployment/tests/test_scheduling.py @@ -109,6 +109,19 @@ def _gpu_device(*, name: str, claim: Literal["DRA", "Synthetic"], count: int, me ) +def _shared_gpu_device(*, count: int, memory: str) -> icv1alpha1.Device: + """A gpu.nvidia.com GPU in a pool that several claims can allocate at once, each consuming part of its memory.""" + return icv1alpha1.Device( + name="gpu", + claim="DRA", + driver="gpu.nvidia.com", + deviceClassName="gpu.nvidia.com", + count=count, + allowMultipleAllocations=True, + capacity={"memory": icv1alpha1.Capacity(value=memory)}, + ) + + def _nic_device(*, link_type: str) -> icv1alpha1.Device: """A Synthetic NIC in a pool, which a nodeSelector can match but nothing claims.""" return icv1alpha1.Device( @@ -6990,6 +7003,575 @@ def _candidate( fill=True, want=[], ), + # Pools whose GPUs allow multiple allocations. A request that sets + # capacity.requests is placed by the capacity each GPU has free, so several + # replicas can share one GPU. Every pool's GPU has 80Gi of memory. + Case( + name="SharedGPUFits", + reason="A replica requesting 24Gi of a GPU's memory fits a GPU another claim holds with 56Gi free, though the node has no GPU free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice( + freeDevices={"gpu.nvidia.com": 0}, + sharedDevices={ + "gpu.nvidia.com": [icv1alpha1.SharedDevice(freeCapacity={"memory": "56Gi"})] + }, + nodes=1, + ), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-shared", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[_MEM_20], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + ), + ], + ), + ], + ), + Case( + name="SharedGPUFull", + reason="A replica requesting 24Gi of a GPU's memory doesn't fit a node whose only GPU another claim holds with 16Gi free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice( + freeDevices={"gpu.nvidia.com": 0}, + sharedDevices={ + "gpu.nvidia.com": [icv1alpha1.SharedDevice(freeCapacity={"memory": "16Gi"})] + }, + nodes=1, + ), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="SharedGPUPacksReplicas", + reason="Replicas requesting 24Gi each pack three to one free 80Gi GPU, and a fourth doesn't fit.", + deployment=_deployment( + replicas=4, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-shared", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[_MEM_20], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=1, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-shared", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[_MEM_20], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + ), + ], + ), + _candidate( + name="cluster-a", + index=2, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-shared", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[_MEM_20], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + ), + ], + ), + ], + ), + Case( + name="DistinctSharedDevices", + reason="A replica requesting 24Gi on each of two GPUs doesn't fit a node with one GPU that has room for two such shares.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=2, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice( + freeDevices={"gpu.nvidia.com": 0}, + sharedDevices={ + "gpu.nvidia.com": [icv1alpha1.SharedDevice(freeCapacity={"memory": "56Gi"})] + }, + nodes=1, + ), + ], + devices=[_shared_gpu_device(count=2, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="SharedPoolWholeGPU", + reason="A replica that sets no capacity.requests takes a whole GPU, so it doesn't fit a node whose only GPU another claim holds, however much of it is free.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[mdv1alpha1.Device(name="gpu", count=1, selectors=[mdv1alpha1.Selector(cel=_MEM_20)])], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice( + freeDevices={"gpu.nvidia.com": 0}, + sharedDevices={ + "gpu.nvidia.com": [icv1alpha1.SharedDevice(freeCapacity={"memory": "79Gi"})] + }, + nodes=1, + ), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="CapacityNeedsSharing", + reason="A request that sets capacity.requests doesn't match a GPU whose class doesn't allow multiple allocations.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_gpu_device(name="gpu", claim="DRA", count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + Case( + name="CapacityRequestExceedsGPU", + reason="A request for more memory than the GPU's class declares doesn't match it.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "96Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[], + ), + # The pool's histogram doesn't yet account for the other replica, which + # holds 48Gi of the pool's one GPU, so the ledger charges it. + Case( + name="SharedGPUUnsettledReplica", + reason="A replica requesting 48Gi doesn't fit a GPU an unsettled replica of another deployment has taken 48Gi of.", + deployment=_deployment( + replicas=1, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "48Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", + nodes=1, + observedNodes=1, + nodesByFreeDevices=[ + icv1alpha1.NodesByFreeDevice(freeDevices={"gpu.nvidia.com": 1}, nodes=1), + ], + devices=[_shared_gpu_device(count=1, memory="80Gi")], + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[ + _replica( + name="other-model-cluster-a-0", + deployment="other-model", + cluster="cluster-a", + index=0, + copies=1, + pods_scheduled=False, + pools=None, + members=[ + _replica_member( + role="Standalone", + worker_nodes=None, + pool="gpu-shared", + device_requests=[ + mrv1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[mrv1alpha1.Selector(cel=_MEM_20)], + capacity=mrv1alpha1.Capacity(requests={"memory": "48Gi"}), + ) + ], + ) + ], + ) + ], + fill=True, + want=[], + ), + Case( + name="SharedGPUUnobservedPool", + reason="On a pool the agent doesn't report on, a replica requesting a share of a GPU still takes a whole node, so a one-node pool hosts one replica.", + deployment=_deployment( + replicas=2, + copies=1, + members=[ + _member( + role="Standalone", + worker_nodes=None, + devices=[ + mdv1alpha1.Device( + name="gpu", + count=1, + selectors=[mdv1alpha1.Selector(cel=_MEM_20)], + capacity=mdv1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + ) + ], + tolerations=None, + ), + clusters=[ + _cluster( + name="cluster-a", + gateway_hostname="cluster-a.clusters.example.com", + ready=True, + pools=[ + icv1alpha1.SchedulablePool( + name="gpu-shared", nodes=1, devices=[_shared_gpu_device(count=1, memory="80Gi")] + ) + ], + taints=None, + placement_labels=None, + ) + ], + all_replicas=[], + fill=True, + want=[ + _candidate( + name="cluster-a", + index=0, + gateway_hostname="cluster-a.clusters.example.com", + placement_labels={}, + members=[ + scheduling.MemberPlacement( + role="Standalone", + pool="gpu-shared", + device_requests=[ + scheduling.DeviceRequest( + name="gpu", + device_class_name="gpu.nvidia.com", + count=1, + cel_selectors=[_MEM_20], + capacity_requests={"memory": "24Gi"}, + memory_fraction="0.3", + ) + ], + ), + ], + ), + ], + ), ] diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index 9fa92e359..aa4e7e619 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -20,6 +20,7 @@ """ import hashlib +import json import re from collections.abc import Mapping from typing import Protocol @@ -425,8 +426,9 @@ def grove_leader_address_env() -> dict: # Response resource keys. A replica's HTTPRoute keeps a stable key; each engine's # workload gets a key scoped by its backend and engine, and each member's claim -# a member-scoped one (the engine name plus the member role) so a multi-engine -# replica's resources don't collide in the response map. +# a member-scoped one (the engine name plus the member role, and a hash of the +# claim's spec - see claim_key) so a multi-engine replica's resources don't +# collide in the response map. ROUTE_KEY = "model-route" _WORKLOAD_KEY = "workload" _SCALING_GROUP_KEY = "scaling-group" @@ -487,8 +489,13 @@ def claim_key(engine: v1alpha1.Engine, member: v1alpha1.Member) -> str: devices through the same template (a template stamps a fresh claim per pod), but an engine's members may claim different devices, or none. The member role disambiguates - an engine has at most one member per role. + + The key carries the hash of the template's spec, as its name does (see + claim_template_name). A changed spec is then a new composed resource, and + Crossplane deletes the Object composing the old template rather than + trying to update it. """ - return f"{_CLAIM_KEY}-{engine.name}-{member_role(member)}" + return f"{_CLAIM_KEY}-{engine.name}-{member_role(member)}-{_claim_spec_hash(member)}" def workload_keys(replica: v1alpha1.ModelReplica, stack: str) -> list[str]: @@ -813,13 +820,26 @@ def claim_template_name(replica: v1alpha1.ModelReplica, engine: v1alpha1.Engine, Per-replica, per-engine, per-member-role: derived from the same parts as engine_name (flat, not nested through engine_name's already-hashed result, - so the name reads replica-engine-role-devices-hash) so concurrent replicas - of the same deployment on one cluster stay distinct. One template serves - every pod of the member - a template stamps a fresh claim per pod - but an - engine's members may claim different devices, so each claiming member gets - its own. + so the name reads replica-engine-role-devices-spec-hash) so concurrent + replicas of the same deployment on one cluster stay distinct. One template + serves every pod of the member - a template stamps a fresh claim per pod - + but an engine's members may claim different devices, so each claiming + member gets its own. + + A ResourceClaimTemplate's spec is immutable, so the name includes a hash of + the spec: a changed device request composes a new template under a new + name, and the member's pods, which reference the template by this name, + roll onto it. child_name truncates the name to a DNS label. """ - return resource.child_name(_name(replica.metadata), engine.name, member_role(member), _POD_CLAIM_NAME) + return resource.child_name( + _name(replica.metadata), engine.name, member_role(member), _POD_CLAIM_NAME, _claim_spec_hash(member) + ) + + +def _claim_spec_hash(member: v1alpha1.Member) -> str: + """A short hash of the spec of a member's ResourceClaimTemplate.""" + spec = json.dumps(claim_template_spec(member), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(spec.encode()).hexdigest()[:8] def engine_resources(container: v1alpha1.Container, *, claim: bool) -> dict: @@ -952,10 +972,24 @@ def resource_claim_template( Each resolved device request (stamped by compose-model-deployment from the matched InferenceClass claim: DRA devices) becomes one DeviceRequest carrying - its DeviceClass, count, and CEL selectors verbatim. Only called for a member - with device requests; a claimless member composes no template. One template - serves every pod of the member, and DRA stamps a fresh claim per pod. + its DeviceClass, count, CEL selectors, and capacity requests verbatim. Only + called for a member with device requests; a claimless member composes no + template. One template serves every pod of the member, and DRA stamps a + fresh claim per pod. """ + return wrap_object( + provider_config, + { + "apiVersion": _DRA_API_VERSION, + "kind": "ResourceClaimTemplate", + "metadata": {"name": claim_template_name(replica, engine, member), "namespace": remote_namespace(replica)}, + "spec": claim_template_spec(member), + }, + ) + + +def claim_template_spec(member: v1alpha1.Member) -> dict: + """The spec of a member's ResourceClaimTemplate: one DeviceRequest per resolved device request.""" # Callers (the backends) gate this on `if member.deviceRequests`, so it's # only reached for a member that claims devices; a claimless member composes # no template. @@ -966,6 +1000,10 @@ def resource_claim_template( selectors = [{"cel": {"expression": s.cel}} for s in (r.selectors or []) if s.cel] if selectors: exactly["selectors"] = selectors + if r.capacity: + # A share of a device that allows multiple allocations, as the + # deployment's nodeSelector requested it. + exactly["capacity"] = {"requests": dict(r.capacity.requests)} if r.deviceClassName == _CPU_DEVICE_CLASS: # dra-driver-cpu's grouped device holds its cores as consumable # capacity, and a claim with no capacity request consumes the @@ -981,13 +1019,4 @@ def resource_claim_template( assert container.resources is not None and container.resources.requests is not None exactly["capacity"] = {"requests": {_CPU_CAPACITY: str(container.resources.requests["cpu"])}} device_requests.append({"name": r.name, "exactly": exactly}) - - return wrap_object( - provider_config, - { - "apiVersion": _DRA_API_VERSION, - "kind": "ResourceClaimTemplate", - "metadata": {"name": claim_template_name(replica, engine, member), "namespace": remote_namespace(replica)}, - "spec": {"spec": {"devices": {"requests": device_requests}}}, - }, - ) + return {"spec": {"devices": {"requests": device_requests}}} diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index e07da1261..fb3f20d9f 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -124,6 +124,16 @@ class RemoteNamespaceCase: want: str +@dataclasses.dataclass +class ClaimTemplateNameCase: + """A test case for base.claim_template_name, of the replica's only member.""" + + name: str + reason: str + replica: v1alpha1.ModelReplica + want: str + + @dataclasses.dataclass class LwsNameCase: """A test case for base.lws_name, of the replica's only engine.""" @@ -826,7 +836,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -854,7 +864,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # A sidecar takes 8000, so the engine declares its http port elsewhere, and @@ -923,7 +935,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -957,7 +969,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # Containers set their own resources; the engine's sit beside its device @@ -1032,7 +1046,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -1072,7 +1086,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # An unnamed port 8000 on the engine is its http port, not a second entry. @@ -1133,7 +1149,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -1162,7 +1178,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # See #378. @@ -1226,7 +1244,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -1256,7 +1274,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # Two replicas of one deployment on the same cluster, this one and @@ -1314,7 +1334,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="dep-clusterA-main-3d1d5", - claim_template_name="dep-clusterA-main-standalone-devices-145eb", + claim_template_name="dep-clusterA-main-standalone-devices-31556731-51560", replicas=1, pod_metadata={ "labels": { @@ -1342,8 +1362,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template( - name="dep-clusterA-main-standalone-devices-145eb", count=1 + "resource-claim-main-standalone-31556731": _claim_template( + name="dep-clusterA-main-standalone-devices-31556731-51560", count=1 ), }, ), @@ -1403,7 +1423,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="dep-clusterB-main-d6c52", - claim_template_name="dep-clusterB-main-standalone-devices-5a8a8", + claim_template_name="dep-clusterB-main-standalone-devices-31556731-80027", replicas=1, pod_metadata={ "labels": { @@ -1431,8 +1451,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template( - name="dep-clusterB-main-standalone-devices-5a8a8", count=1 + "resource-claim-main-standalone-31556731": _claim_template( + name="dep-clusterB-main-standalone-devices-31556731-80027", count=1 ), }, ), @@ -1527,7 +1547,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -1555,10 +1575,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "workload-native-decode": _deployment( name="r-decode-4b27b", - claim_template_name="r-decode-standalone-devices-63392", + claim_template_name="r-decode-standalone-devices-31556731-771a9", replicas=1, pod_metadata={ "labels": { @@ -1586,7 +1608,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "resource-claim-decode-standalone-31556731": _claim_template( + name="r-decode-standalone-devices-31556731-771a9", count=1 + ), }, ), # resources.claims is a list-map keyed on name alone, so N device requests @@ -1639,7 +1663,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-95ac100f-9f9ba", replicas=1, pod_metadata={ "labels": { @@ -1669,7 +1693,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ), # Written out because it's the only claim template that asks for # more than GPUs, which is all _claim_template builds. - "resource-claim-main-standalone": { + "resource-claim-main-standalone-95ac100f": { "spec": { "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "cluster-a-pc"}, "readiness": {"policy": "SuccessfulCreate"}, @@ -1677,7 +1701,10 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "manifest": { "apiVersion": "resource.k8s.io/v1", "kind": "ResourceClaimTemplate", - "metadata": {"name": "r-main-standalone-devices-f456f", "namespace": "mp-ml-team-51733"}, + "metadata": { + "name": "r-main-standalone-devices-95ac100f-9f9ba", + "namespace": "mp-ml-team-51733", + }, "spec": { "spec": { "devices": { @@ -1758,7 +1785,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -1796,7 +1823,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # Kubernetes expands $(VAR) left to right, so Modelplane's own entries must @@ -1855,7 +1884,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -1896,7 +1925,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # A Standalone engine on a Dynamo cluster with a cache is as valid a P2P @@ -1961,7 +1992,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -2027,7 +2058,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, ), # There's no flag injection or bootstrap. The worker addresses the leader @@ -2182,7 +2215,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -2242,7 +2275,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -2267,8 +2300,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # A Grove leader behind a sidecar, and a worker carrying its own ports and @@ -2426,7 +2463,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -2488,7 +2525,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -2513,8 +2550,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), BuildCase( @@ -2656,7 +2697,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -2716,7 +2757,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -2741,8 +2782,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Multi-NIC RDMA nodes need VLLM_HOST_IP from status.podIP so the engine @@ -2896,7 +2941,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -2956,7 +3001,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -2981,8 +3026,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Grove propagates a clique's labels and annotations to its pods (#378). @@ -3136,7 +3185,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -3198,7 +3247,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -3223,8 +3272,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # A coordinator-only leader, like a vLLM DP head running @@ -3410,7 +3463,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -3435,7 +3488,9 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # The scheduler may split a gang across pools when no single pool satisfies @@ -3578,7 +3633,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -3638,7 +3693,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -3663,8 +3718,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Pointing HF_HUB_CACHE at the mount lets each clique's own --model= @@ -3828,7 +3887,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -3920,7 +3979,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -3945,8 +4004,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), BuildCase( @@ -4121,7 +4184,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -4213,7 +4276,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -4238,8 +4301,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Each pod publishes itself as a source independently, so the worker needs @@ -4420,7 +4487,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-leader-devices-f58c6", + "resourceClaimTemplateName": "r-main-leader-devices-8b2f53e4-bf5db", } ], "tolerations": [ @@ -4516,7 +4583,7 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "resourceClaims": [ { "name": "devices", - "resourceClaimTemplateName": "r-main-worker-devices-99b8a", + "resourceClaimTemplateName": "r-main-worker-devices-8b2f53e4-fc752", } ], "tolerations": [ @@ -4541,8 +4608,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # The worker followers never serve, so without the serving label the @@ -4624,8 +4695,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -4668,8 +4739,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # LWS names a LeaderWorkerSet's headless Service after it, so its webhook @@ -4748,8 +4823,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="lws-qwen2-5-main-b2784-6d975", serving="qwen2.5", - leader_claim_template_name="qwen2.5-main-leader-devices-5743a", - worker_claim_template_name="qwen2.5-main-worker-devices-4d4ac", + leader_claim_template_name="qwen2.5-main-leader-devices-8b2f53e4-0eb95", + worker_claim_template_name="qwen2.5-main-worker-devices-8b2f53e4-1949b", replicas=1, size=2, leader_containers=[ @@ -4792,8 +4867,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="qwen2.5-main-leader-devices-5743a", count=8), - "resource-claim-main-worker": _claim_template(name="qwen2.5-main-worker-devices-4d4ac", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="qwen2.5-main-leader-devices-8b2f53e4-0eb95", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="qwen2.5-main-worker-devices-8b2f53e4-1949b", count=8 + ), }, ), # Each member has its own template, so a container the user adds to the @@ -4875,8 +4954,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -4920,8 +4999,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # A Worker doesn't serve: its ports pass through and nothing probes it. @@ -5004,8 +5087,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -5050,8 +5133,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Modelplane is unopinionated about the engine: each member's command passes @@ -5150,8 +5237,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -5204,8 +5291,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), BuildCase( @@ -5278,8 +5369,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=2, size=4, leader_containers=[ @@ -5322,8 +5413,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), # Only the native and Grove backends wire ModelExpress, and only on Dynamo, @@ -5399,8 +5494,8 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -5463,8 +5558,12 @@ def _sorted(*, objects: dict[str, dict]) -> dict[str, dict]: }, ], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, ), ] @@ -5584,8 +5683,8 @@ def test_cpu_claim_capacity() -> None: def test_gpu_claim_uncapped() -> None: - """A GPU claim carries no capacity request: GPUs bind whole devices, and - a capacity request is the CPU driver's contract, not DRA's generally.""" + """A GPU claim whose device request carries no capacity requests carries + none either, so it takes whole devices.""" replica = _gpu_replica() engine = replica.spec.engines[0] obj = base.resource_claim_template(replica, engine, engine.members[0], "cluster-a-pc") @@ -5594,6 +5693,136 @@ def test_gpu_claim_uncapped() -> None: assert "capacity" not in requests[0]["exactly"] +def test_gpu_claim_share() -> None: + """A GPU claim carries its device request's capacity requests verbatim, + so DRA allocates it a share of a device that allows multiple + allocations.""" + replica = _replica( + name="r", + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="shared", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=1, + selectors=[v1alpha1.Selector(cel='device.driver == "gpu.nvidia.com"')], + capacity=v1alpha1.Capacity(requests={"memory": "24Gi"}), + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")] + ) + ), + ) + ], + ) + ], + ) + engine = replica.spec.engines[0] + obj = base.resource_claim_template(replica, engine, engine.members[0], "cluster-a-pc") + got = obj.model_dump(exclude_unset=True, by_alias=True) + requests = got["spec"]["forProvider"]["manifest"]["spec"]["spec"]["devices"]["requests"] + assert requests[0]["exactly"]["capacity"] == {"requests": {"memory": "24Gi"}} + + +def _claim_replica(*, name: str, count: int, memory: str | None) -> v1alpha1.ModelReplica: + """A replica whose one Standalone member claims count GPUs of at least 80Gi, and memory of each if given.""" + return _replica( + name=name, + namespace="ml-team", + cluster_name="cluster-a", + labels=None, + model_cache_ref=None, + serving=None, + engines=[ + v1alpha1.Engine( + name="main", + copies=1, + members=[ + v1alpha1.Member( + role="Standalone", + nodePoolName="frontier", + deviceRequests=[ + v1alpha1.DeviceRequest( + name="gpu", + deviceClassName="gpu.nvidia.com", + count=count, + selectors=[ + v1alpha1.Selector( + cel='device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("80Gi")) >= 0' + ) + ], + capacity=v1alpha1.Capacity(requests={"memory": memory}) if memory is not None else None, + ) + ], + template=v1alpha1.Template( + spec=v1alpha1.Spec( + containers=[v1alpha1.Container(name="engine", image="vllm/vllm-openai:latest")] + ) + ), + ) + ], + ) + ], + ) + + +# A ResourceClaimTemplate's spec is immutable, so a member whose device +# requests change needs a template with a new name. +CLAIM_TEMPLATE_NAME_CASES = [ + ClaimTemplateNameCase( + name="WholeDevice", + reason="A member claiming one whole GPU gets a template named for its replica, engine, role and spec.", + replica=_claim_replica(name="r", count=1, memory=None), + want="r-main-standalone-devices-31556731-b687f", + ), + ClaimTemplateNameCase( + name="CapacityRequest", + reason="Requesting a share of the GPU's memory changes the spec, and so the template's name.", + replica=_claim_replica(name="r", count=1, memory="24Gi"), + want="r-main-standalone-devices-ee595852-15ae6", + ), + ClaimTemplateNameCase( + name="CapacityChanged", + reason="Requesting a different share changes the template's name again.", + replica=_claim_replica(name="r", count=1, memory="32Gi"), + want="r-main-standalone-devices-b3234b61-247a2", + ), + ClaimTemplateNameCase( + name="CountChanged", + reason="Claiming two GPUs instead of one changes the template's name.", + replica=_claim_replica(name="r", count=2, memory=None), + want="r-main-standalone-devices-9dfa9360-0c2f8", + ), + ClaimTemplateNameCase( + name="LongReplicaName", + reason="A replica name of 63 characters is truncated so the template's name is a 63-character DNS label.", + replica=_claim_replica(name="a" * 63, count=1, memory=None), + want="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-9ff43", + ), +] + + +@pytest.mark.parametrize("case", CLAIM_TEMPLATE_NAME_CASES, ids=lambda case: case.name) +def test_claim_template_name(case: ClaimTemplateNameCase) -> None: + """claim_template_name names a member's template for its replica, engine, role and spec.""" + engine = case.replica.spec.engines[0] + assert base.claim_template_name(case.replica, engine, engine.members[0]) == case.want, case.reason + + INVALID_CPU_CLAIM_CASES = [ InvalidCpuClaimCase( name="CpuRequestSet", @@ -6067,7 +6296,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -6095,10 +6324,12 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "workload-native-decode": _deployment( name="r-decode-4b27b", - claim_template_name="r-decode-standalone-devices-63392", + claim_template_name="r-decode-standalone-devices-31556731-771a9", replicas=1, pod_metadata={ "labels": { @@ -6126,7 +6357,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "resource-claim-decode-standalone-31556731": _claim_template( + name="r-decode-standalone-devices-31556731-771a9", count=1 + ), }, replica=_replica( name="r", @@ -6237,7 +6470,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -6268,10 +6501,12 @@ def test_cache_env(case: CacheEnvCase) -> None: {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "workload-native-decode": _deployment( name="r-decode-4b27b", - claim_template_name="r-decode-standalone-devices-63392", + claim_template_name="r-decode-standalone-devices-31556731-771a9", replicas=1, pod_metadata={ "labels": { @@ -6302,7 +6537,9 @@ def test_cache_env(case: CacheEnvCase) -> None: {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), - "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "resource-claim-decode-standalone-31556731": _claim_template( + name="r-decode-standalone-devices-31556731-771a9", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -6333,7 +6570,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -6361,10 +6598,12 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "workload-native-decode": _deployment( name="r-decode-4b27b", - claim_template_name="r-decode-standalone-devices-63392", + claim_template_name="r-decode-standalone-devices-31556731-771a9", replicas=1, pod_metadata={ "labels": { @@ -6392,7 +6631,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "resource-claim-decode-standalone-31556731": _claim_template( + name="r-decode-standalone-devices-31556731-771a9", count=1 + ), }, replica=_replica( name="r", @@ -6510,7 +6751,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -6541,10 +6782,12 @@ def test_cache_env(case: CacheEnvCase) -> None: {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "workload-native-decode": _deployment( name="r-decode-4b27b", - claim_template_name="r-decode-standalone-devices-63392", + claim_template_name="r-decode-standalone-devices-31556731-771a9", replicas=1, pod_metadata={ "labels": { @@ -6575,7 +6818,9 @@ def test_cache_env(case: CacheEnvCase) -> None: {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), - "resource-claim-decode-standalone": _claim_template(name="r-decode-standalone-devices-63392", count=1), + "resource-claim-decode-standalone-31556731": _claim_template( + name="r-decode-standalone-devices-31556731-771a9", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -6937,7 +7182,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -6965,7 +7210,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "scaling-group-decode": _scaling_group(pcs_name="r-decode-4b27b"), "workload-grove-decode": _pod_clique_set( name="r-decode-4b27b", @@ -7249,7 +7496,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-prefill": _deployment( name="r-prefill-d90b0", - claim_template_name="r-prefill-standalone-devices-f62af", + claim_template_name="r-prefill-standalone-devices-31556731-d5dbc", replicas=1, pod_metadata={ "labels": { @@ -7280,7 +7527,9 @@ def test_cache_env(case: CacheEnvCase) -> None: {"name": "dshm", "emptyDir": {"medium": "Memory"}}, ], ), - "resource-claim-prefill-standalone": _claim_template(name="r-prefill-standalone-devices-f62af", count=1), + "resource-claim-prefill-standalone-31556731": _claim_template( + name="r-prefill-standalone-devices-31556731-d5dbc", count=1 + ), "scaling-group-decode": _scaling_group(pcs_name="r-decode-4b27b"), "workload-grove-decode": _pod_clique_set( name="r-decode-4b27b", @@ -7463,7 +7712,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7491,7 +7740,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -7541,7 +7792,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7569,7 +7820,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -7619,7 +7872,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7647,7 +7900,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -7697,7 +7952,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7725,7 +7980,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -7775,7 +8032,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7803,7 +8060,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -7853,7 +8112,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7881,7 +8140,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -7934,7 +8195,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -7962,7 +8223,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -8043,7 +8306,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -8071,7 +8334,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -8147,7 +8412,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -8175,7 +8440,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -8241,7 +8508,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -8269,7 +8536,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -8319,7 +8588,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -8349,7 +8618,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -8399,7 +8670,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=1, pod_metadata={ "labels": { @@ -8429,7 +8700,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -8486,7 +8759,7 @@ def test_cache_env(case: CacheEnvCase) -> None: composed={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=2, pod_metadata={ "labels": { @@ -8514,7 +8787,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), }, replica=_replica( name="r", @@ -8564,7 +8839,7 @@ def test_cache_env(case: CacheEnvCase) -> None: want={ "workload-native-main": _deployment( name="r-main-bb4e3", - claim_template_name="r-main-standalone-devices-f456f", + claim_template_name="r-main-standalone-devices-31556731-b687f", replicas=2, pod_metadata={ "labels": { @@ -8592,7 +8867,9 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="r-main-standalone-devices-f456f", count=1), + "resource-claim-main-standalone-31556731": _claim_template( + name="r-main-standalone-devices-31556731-b687f", count=1 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -8649,8 +8926,8 @@ def test_cache_env(case: CacheEnvCase) -> None: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -8703,8 +8980,12 @@ def test_cache_env(case: CacheEnvCase) -> None: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), }, replica=_replica( name="r", @@ -8791,8 +9072,8 @@ def test_cache_env(case: CacheEnvCase) -> None: "workload-llmd-main": _leader_worker_set( name="r-main-bb4e3", serving="r", - leader_claim_template_name="r-main-leader-devices-f58c6", - worker_claim_template_name="r-main-worker-devices-99b8a", + leader_claim_template_name="r-main-leader-devices-8b2f53e4-bf5db", + worker_claim_template_name="r-main-worker-devices-8b2f53e4-fc752", replicas=1, size=2, leader_containers=[ @@ -8845,8 +9126,12 @@ def test_cache_env(case: CacheEnvCase) -> None: ], worker_volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-leader": _claim_template(name="r-main-leader-devices-f58c6", count=8), - "resource-claim-main-worker": _claim_template(name="r-main-worker-devices-99b8a", count=8), + "resource-claim-main-leader-8b2f53e4": _claim_template( + name="r-main-leader-devices-8b2f53e4-bf5db", count=8 + ), + "resource-claim-main-worker-8b2f53e4": _claim_template( + name="r-main-worker-devices-8b2f53e4-fc752", count=8 + ), "inference-pool": _inference_pool(name="r-pool", serving="r", picker="r-epp"), "model-route": _route(name="r", path="/ml-team/r/", pool="r-pool"), "epp-serviceaccount": _epp_service_account(name="r-epp"), @@ -8931,7 +9216,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template( + "resource-claim-main-standalone-31556731": _claim_template( name="qwen2.5-7b-abcde-main-standalone-devices-771a9", count=1 ), }, @@ -9011,7 +9296,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template( + "resource-claim-main-standalone-31556731": _claim_template( name="qwen2.5-7b-abcde-main-standalone-devices-771a9", count=1 ), "inference-pool": _inference_pool( @@ -9102,7 +9387,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="a" * 57 + "-05ced", count=1), + "resource-claim-main-standalone-31556731": _claim_template(name="a" * 57 + "-05ced", count=1), }, replica=_replica( name="a" * 57 + "-abcde", @@ -9180,7 +9465,7 @@ def test_cache_env(case: CacheEnvCase) -> None: ], volumes=[{"name": "dshm", "emptyDir": {"medium": "Memory"}}], ), - "resource-claim-main-standalone": _claim_template(name="a" * 57 + "-05ced", count=1), + "resource-claim-main-standalone-31556731": _claim_template(name="a" * 57 + "-05ced", count=1), "inference-pool": _inference_pool( name="a" * 57 + "-abcde-pool", serving="a" * 57 + "-abcde", picker="epp-" + "a" * 53 + "-2dd65" ), diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index 5e37673e1..4b625634b 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -1064,7 +1064,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-prefill-10d7e", - claim_template_name="test-replica-prefill-standalone-devices-51762", + claim_template_name="test-replica-prefill-standalone-devices-31556731-fa720", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "prefill", @@ -1076,14 +1076,14 @@ def _to_dict(msg: message.Message) -> dict: "llm-d.ai/role": "prefill", }, ), - "resource-claim-prefill-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-prefill-standalone-devices-51762" + "resource-claim-prefill-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-prefill-standalone-devices-31556731-fa720" ), "workload-native-decode": _deployment( copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-decode-44185", - claim_template_name="test-replica-decode-standalone-devices-e863a", + claim_template_name="test-replica-decode-standalone-devices-31556731-f60b1", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "decode", @@ -1095,8 +1095,8 @@ def _to_dict(msg: message.Message) -> dict: "llm-d.ai/role": "decode", }, ), - "resource-claim-decode-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-decode-standalone-devices-e863a" + "resource-claim-decode-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-decode-standalone-devices-31556731-f60b1" ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), @@ -1196,7 +1196,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1207,8 +1207,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1293,7 +1293,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1304,8 +1304,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1455,7 +1455,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1466,8 +1466,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1602,7 +1602,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1613,8 +1613,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1691,7 +1691,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1702,8 +1702,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1793,7 +1793,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1804,8 +1804,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_UNSPECIFIED), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_UNSPECIFIED, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -1934,7 +1934,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=True, ready_replicas=1), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), }, ), required_resources={ @@ -1949,7 +1949,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_TRUE, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -1960,8 +1960,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_UNSPECIFIED), "epp": _epp( @@ -2017,7 +2017,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=True, ready_replicas=1), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2040,7 +2040,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_TRUE, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2051,8 +2051,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2108,7 +2108,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=True, ready_replicas=1), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=False), "epp-config": _observed_object(ready=True), @@ -2131,7 +2131,7 @@ def _to_dict(msg: message.Message) -> dict: copies=1, ready=fnv1.READY_TRUE, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2142,8 +2142,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2196,7 +2196,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=False, ready_replicas=1), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2219,7 +2219,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, copies=2, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2230,8 +2230,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2277,7 +2277,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=False, ready_replicas=0), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2300,7 +2300,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, copies=2, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2311,8 +2311,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2361,7 +2361,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=False, ready_replicas=2), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2384,7 +2384,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_UNSPECIFIED, copies=2, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2395,8 +2395,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2442,7 +2442,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=True, ready_replicas=2), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2465,7 +2465,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, copies=2, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2476,8 +2476,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( @@ -2526,7 +2526,7 @@ def _to_dict(msg: message.Message) -> dict: resources={ "workload-native-main": _observed_deployment(ready=True, ready_replicas=2), "model-route": _observed_object(ready=True), - "resource-claim-main-standalone": _observed_object(ready=True), + "resource-claim-main-standalone-31556731": _observed_object(ready=True), "inference-pool": _observed_object(ready=True), "epp": _observed_object(ready=True), "epp-config": _observed_object(ready=True), @@ -2549,7 +2549,7 @@ def _to_dict(msg: message.Message) -> dict: ready=fnv1.READY_TRUE, copies=2, name="test-replica-main-6b608", - claim_template_name="test-replica-main-standalone-devices-e609d", + claim_template_name="test-replica-main-standalone-devices-31556731-6a31b", labels={ "modelplane.ai/deployment": "my-deployment", "modelplane.ai/engine": "main", @@ -2560,8 +2560,8 @@ def _to_dict(msg: message.Message) -> dict: }, ), "model-route": _route(ready=fnv1.READY_TRUE), - "resource-claim-main-standalone": _claim_template( - ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-e609d" + "resource-claim-main-standalone-31556731": _claim_template( + ready=fnv1.READY_TRUE, name="test-replica-main-standalone-devices-31556731-6a31b" ), "inference-pool": _inference_pool(ready=fnv1.READY_TRUE), "epp": _epp( diff --git a/schemas/.lock.json b/schemas/.lock.json index 03808b4c1..ffeb46e9a 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,6 +1,6 @@ { "packages": { - "fs://apis": "c9210e3c0469d7ffccb38bc7fe8975d7ea2d58e880e7e5d94b9e615a6013a1fe", + "fs://apis": "7a1421e7f7401f6f73741fb9fa9a97c1a798f089edf3c67b7ca72f304a8beec6", "git://https://github.com/crossplane/crossplane/cluster/crds": "14c53aac914d393b5bcc00962fbc90d4be5d35bc", "git://https://github.com/modelplaneai/cluster-agent/cluster/charts/cluster-agent/files/crds": "a833d3f6f7d9a6ac6083e7453375b58b65ee741a", "xpkg://xpkg.upbound.io/crossplane-contrib/provider-upcloud:v1.1.0": "sha256:ef449e492e8277974c502672765ea83e479a167738f4fe3e59577f67b8fda622", diff --git a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py index 1e2b46027..0158db328 100644 --- a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py @@ -60,6 +60,10 @@ class Capacity(BaseModel): class Device(BaseModel): + allowMultipleAllocations: bool | None = None + """ + Whether several claims can allocate one of these devices at once, each consuming part of its capacity, as DRA's consumable capacity (KEP-5075) allows. Set it only when the DRA driver publishes the device with allowMultipleAllocations. A ModelDeployment request that sets capacity.requests matches only a device that allows multiple allocations, and is scheduled by the capacity its device has free. Only a claim: DRA device can set it. Omitted, the scheduler counts the device whole. + """ attributes: dict[str, Attributes] | None = Field(None, max_length=32) """ DRA-style typed attributes for this device. Keys are bare names (e.g. architecture); the domain comes from the device's driver. Each value sets exactly one typed field. diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 608427b3e..bc7f24018 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -438,6 +438,7 @@ class Capacity(BaseModel): class Device(BaseModel): + allowMultipleAllocations: bool | None = None attributes: dict[str, Attributes] | None = Field(None, max_length=32) capacity: dict[str, Capacity] | None = Field(None, max_length=32) claim: Literal['DRA', 'Synthetic'] | None = None @@ -447,6 +448,13 @@ class Device(BaseModel): name: constr(max_length=63) +class SharedDevice(BaseModel): + freeCapacity: dict[str, constr(max_length=32)] | None = Field(None, max_length=32) + """ + The device's capacity no claim has consumed, keyed by the names its driver publishes capacity under. + """ + + class NodesByFreeDevice(BaseModel): freeDevices: dict[str, conint(ge=0)] | None = Field(None, max_length=32) """ @@ -456,6 +464,10 @@ class NodesByFreeDevice(BaseModel): """ Nodes with these free devices. """ + sharedDevices: dict[str, list[SharedDevice]] | None = Field(None, max_length=32) + """ + The free capacity of each of the nodes' devices that allows multiple allocations and that a claim has allocated, keyed by DeviceClass. Such a device doesn't count in freeDevices. One no claim has allocated does, with all of its capacity free. + """ class SchedulablePool(BaseModel): diff --git a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py index 215a64383..f829ffb5b 100644 --- a/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modeldeployment/v1alpha1.py @@ -52,6 +52,19 @@ class ClusterSelector(BaseModel): matchLabels: dict[str, str] | None = None +class Capacity(BaseModel): + requests: dict[ + str, + constr( + pattern=r'^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$', + max_length=32, + ), + ] = Field(..., max_length=8, min_length=1) + """ + The capacity to consume on each device, keyed by the names the device declares capacity under (e.g. memory); values are Kubernetes Quantities. When exactly one of a member's requests asks for memory, its engine container gets MODELPLANE_DEVICE_MEMORY_FRACTION: the requested memory over the memory the device's class declares. + """ + + class Selector(BaseModel): cel: constr(min_length=1, max_length=10240) | None = None """ @@ -60,6 +73,10 @@ class Selector(BaseModel): class Device(BaseModel): + capacity: Capacity | None = None + """ + The share of each device's capacity the request consumes, mirroring a DRA DeviceRequest's capacity and carried through to the ResourceClaim. Setting it makes the request match only a device whose InferenceClass sets allowMultipleAllocations and declares at least the requested amount of each capacity. The scheduler then places the request on devices with that much capacity free, so several requests can share one device. Omitted, the request takes whole devices. + """ count: conint(ge=1, le=64) | None = 1 """ How many matching devices a node must have. For a GPU request this is the per-node GPU count. diff --git a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py index 1d8946338..a07d7458f 100644 --- a/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/modelreplica/v1alpha1.py @@ -41,11 +41,19 @@ class Crossplane(BaseModel): resourceRefs: list[ResourceRef] | None = None +class Capacity(BaseModel): + requests: dict[str, constr(max_length=32)] = Field(..., max_length=8, min_length=1) + + class Selector(BaseModel): cel: constr(min_length=1, max_length=10240) | None = None class DeviceRequest(BaseModel): + capacity: Capacity | None = None + """ + The capacity to consume on each device, copied verbatim from the nodeSelector request into the DeviceRequest. + """ count: conint(ge=1, le=64) | None = 1 """ How many devices to claim.