Skip to content

Commit 22e85a0

Browse files
committed
fix(vm): validate GPU DRA device configuration
Signed-off-by: Daniil Antoshin <daniil.antoshin@flant.com>
1 parent 1014be1 commit 22e85a0

17 files changed

Lines changed: 396 additions & 11 deletions

File tree

api/core/v1alpha2/virtual_machine.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ type VirtualMachineSpec struct {
121121
USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"`
122122
// List of GPU devices to attach to the virtual machine.
123123
// Devices are requested by GPU model.
124+
// This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass.
124125
// +kubebuilder:validation:MaxItems:=16
125126
// +listType=map
126127
// +listMapKey=name
@@ -506,8 +507,9 @@ type USBDeviceSpecRef struct {
506507
// GPUDeviceSpec requests a GPU device by model.
507508
type GPUDeviceSpec struct {
508509
// A unique GPU device name inside the virtual machine spec.
510+
// The value is used to generate DRA claim and request names.
509511
// +kubebuilder:validation:MinLength:=1
510-
// +kubebuilder:validation:MaxLength:=63
512+
// +kubebuilder:validation:MaxLength:=55
511513
// +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
512514
Name string `json:"name"`
513515
// GPU product name, for example NVIDIA H100.

crds/doc-ru-virtualmachines.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -592,6 +592,7 @@ spec:
592592
description: |
593593
Список GPU-устройств для подключения к виртуальной машине.
594594
Устройства запрашиваются по модели GPU.
595+
Для использования требуется feature gate `GPU` и DeviceClass `gpu.deckhouse.io`.
595596
items:
596597
properties:
597598
model:
@@ -600,6 +601,7 @@ spec:
600601
name:
601602
description: |
602603
Уникальное имя GPU-устройства внутри спецификации виртуальной машины.
604+
Значение используется для генерации имён DRA claim и request.
603605
status:
604606
properties:
605607
blockDeviceRefs:

crds/virtualmachines.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1047,6 +1047,7 @@ spec:
10471047
description: |
10481048
List of GPU devices to attach to the virtual machine.
10491049
Devices are requested by GPU model.
1050+
This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass.
10501051
items:
10511052
type: object
10521053
required:
@@ -1061,11 +1062,12 @@ spec:
10611062
GPU product name, for example NVIDIA H100.
10621063
name:
10631064
minLength: 1
1064-
maxLength: 63
1065+
maxLength: 55
10651066
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
10661067
type: string
10671068
description: |
10681069
A unique GPU device name inside the virtual machine spec.
1070+
The value is used to generate DRA claim and request names.
10691071
status:
10701072
type: object
10711073
properties:

docs/USER_GUIDE.md

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3775,6 +3775,52 @@ spec:
37753775

37763776
As a result, a VM named `clone-database-prod` and a disk named `clone-database-root-prod` will be created.
37773777

3778+
## GPU Devices
3779+
3780+
{{< alert level="warning" >}}
3781+
GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that creates the `gpu.deckhouse.io` `DeviceClass`.
3782+
{{< /alert >}}
3783+
3784+
The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by product model through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource.
3785+
3786+
GPU device passthrough requires:
3787+
3788+
- Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration.
3789+
- The `GPU` feature gate enabled in the `virtualization` module settings.
3790+
- A GPU DRA provider installed in the cluster.
3791+
- The `gpu.deckhouse.io` [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) created by the GPU DRA provider.
3792+
3793+
To enable the module feature gate:
3794+
3795+
```yaml
3796+
apiVersion: deckhouse.io/v1alpha1
3797+
kind: ModuleConfig
3798+
metadata:
3799+
name: virtualization
3800+
spec:
3801+
settings:
3802+
featureGates:
3803+
- GPU
3804+
```
3805+
3806+
To request a GPU device, add `.spec.gpuDevices` to the VM specification:
3807+
3808+
```yaml
3809+
apiVersion: virtualization.deckhouse.io/v1alpha2
3810+
kind: VirtualMachine
3811+
metadata:
3812+
name: linux-vm
3813+
spec:
3814+
# ... other VM settings ...
3815+
gpuDevices:
3816+
- name: gpu0
3817+
model: NVIDIA H100
3818+
```
3819+
3820+
The `name` field must be unique within `.spec.gpuDevices` and can contain up to 55 DNS-label characters. The `model` field must match the GPU product name exposed by the GPU DRA provider in the `device.attributes["gpu.deckhouse.io"].productName` device attribute.
3821+
3822+
Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration.
3823+
37783824
## USB Devices
37793825

37803826
{{< alert level="warning" >}}

docs/USER_GUIDE.ru.md

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3806,6 +3806,52 @@ spec:
38063806

38073807
В результате будет создана ВМ с именем `clone-database-prod` и диск с именем `clone-database-root-prod`.
38083808

3809+
## GPU-устройства
3810+
3811+
{{< alert level="warning" >}}
3812+
Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, создающий `DeviceClass` с именем `gpu.deckhouse.io`.
3813+
{{< /alert >}}
3814+
3815+
Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по модели продукта через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine).
3816+
3817+
Для проброса GPU требуются:
3818+
3819+
- Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера.
3820+
- Feature gate `GPU`, включённый в настройках модуля `virtualization`.
3821+
- Установленный в кластере GPU DRA-провайдер.
3822+
- [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) `gpu.deckhouse.io`, созданный GPU DRA-провайдером.
3823+
3824+
Чтобы включить feature gate модуля:
3825+
3826+
```yaml
3827+
apiVersion: deckhouse.io/v1alpha1
3828+
kind: ModuleConfig
3829+
metadata:
3830+
name: virtualization
3831+
spec:
3832+
settings:
3833+
featureGates:
3834+
- GPU
3835+
```
3836+
3837+
Чтобы запросить GPU-устройство, добавьте `.spec.gpuDevices` в спецификацию ВМ:
3838+
3839+
```yaml
3840+
apiVersion: virtualization.deckhouse.io/v1alpha2
3841+
kind: VirtualMachine
3842+
metadata:
3843+
name: linux-vm
3844+
spec:
3845+
# ... другие настройки ВМ ...
3846+
gpuDevices:
3847+
- name: gpu0
3848+
model: NVIDIA H100
3849+
```
3850+
3851+
Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 55 символов DNS label. Поле `model` должно совпадать с названием продукта GPU, которое GPU DRA-провайдер публикует в атрибуте устройства `device.attributes["gpu.deckhouse.io"].productName`.
3852+
3853+
Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации.
3854+
38093855
## USB-устройства
38103856

38113857
{{< alert level="warning">}}

images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030

3131
const (
3232
GPUNamePrefix = "gpu-"
33+
GPUDeviceClassName = "gpu.deckhouse.io"
3334
GPUResourceClaimTemplateNameSuffixFormat = "-gpu-%s-template"
3435
GPUResourceClaimRequestNamePrefix = "req-gpu-"
3536
)

images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,7 @@ import (
3636
"github.com/deckhouse/virtualization/api/core/v1alpha2"
3737
)
3838

39-
const (
40-
nameGPUResourceClaimHandler = "GPUResourceClaimHandler"
41-
gpuDeviceClassName = "gpu.deckhouse.io"
42-
)
39+
const nameGPUResourceClaimHandler = "GPUResourceClaimHandler"
4340

4441
func NewGPUResourceClaimHandler(client client.Client) *GPUResourceClaimHandler {
4542
return &GPUResourceClaimHandler{client: client}
@@ -83,6 +80,10 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac
8380
continue
8481
}
8582

83+
if !metav1.IsControlledBy(template, vm) {
84+
return reconcile.Result{}, fmt.Errorf("GPU ResourceClaimTemplate %s/%s is not controlled by VirtualMachine %s/%s", template.Namespace, template.Name, vm.Namespace, vm.Name)
85+
}
86+
8687
if reflect.DeepEqual(template.Spec, desiredSpec) {
8788
continue
8889
}
@@ -124,7 +125,7 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1
124125
Requests: []resourcev1.DeviceRequest{{
125126
Name: kvbuilder.GPUResourceClaimRequestName(device.Name),
126127
Exactly: &resourcev1.ExactDeviceRequest{
127-
DeviceClassName: gpuDeviceClassName,
128+
DeviceClassName: kvbuilder.GPUDeviceClassName,
128129
AllocationMode: resourcev1.DeviceAllocationModeExactCount,
129130
Count: 1,
130131
Selectors: []resourcev1.DeviceSelector{{

images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ var _ = Describe("GPUResourceClaimHandler", func() {
5555
Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1))
5656
request := template.Spec.Spec.Devices.Requests[0]
5757
Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName("gpu0")))
58-
Expect(request.Exactly.DeviceClassName).To(Equal(gpuDeviceClassName))
58+
Expect(request.Exactly.DeviceClassName).To(Equal(kvbuilder.GPUDeviceClassName))
5959
Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`))
6060
Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`))
6161
Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`))
@@ -74,4 +74,20 @@ var _ = Describe("GPUResourceClaimHandler", func() {
7474
err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, stored)
7575
Expect(err).To(HaveOccurred())
7676
})
77+
78+
It("should not replace GPU ResourceClaimTemplate owned by another controller", func() {
79+
vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel})
80+
template := &resourcev1.ResourceClaimTemplate{
81+
ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace},
82+
}
83+
fakeClient, _, vmState := setupEnvironment(vm, template)
84+
handler := NewGPUResourceClaimHandler(fakeClient)
85+
86+
_, err := handler.Handle(context.Background(), vmState)
87+
88+
Expect(err).To(HaveOccurred())
89+
stored := &resourcev1.ResourceClaimTemplate{}
90+
Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: template.Name, Namespace: namespace}, stored)).To(Succeed())
91+
Expect(stored.OwnerReferences).To(BeEmpty())
92+
})
7793
})
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
Copyright 2026 Flant JSC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package validators
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
resourcev1 "k8s.io/api/resource/v1"
24+
apierrors "k8s.io/apimachinery/pkg/api/errors"
25+
"k8s.io/component-base/featuregate"
26+
"sigs.k8s.io/controller-runtime/pkg/client"
27+
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
28+
29+
"github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder"
30+
"github.com/deckhouse/virtualization-controller/pkg/featuregates"
31+
"github.com/deckhouse/virtualization/api/core/v1alpha2"
32+
)
33+
34+
type GPUDevicesValidator struct {
35+
client client.Client
36+
featureGate featuregate.FeatureGate
37+
}
38+
39+
func NewGPUDevicesValidator(client client.Client, featureGate featuregate.FeatureGate) *GPUDevicesValidator {
40+
return &GPUDevicesValidator{client: client, featureGate: featureGate}
41+
}
42+
43+
func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) {
44+
return nil, v.validateGPUDevices(ctx, vm)
45+
}
46+
47+
func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, _, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) {
48+
return nil, v.validateGPUDevices(ctx, newVM)
49+
}
50+
51+
func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error {
52+
if len(vm.Spec.GPUDevices) == 0 {
53+
return nil
54+
}
55+
56+
if !v.featureGate.Enabled(featuregates.GPU) {
57+
return fmt.Errorf("GPU device attachment requires the GPU feature gate")
58+
}
59+
60+
deviceClass := &resourcev1.DeviceClass{}
61+
err := v.client.Get(ctx, client.ObjectKey{Name: kvbuilder.GPUDeviceClassName}, deviceClass)
62+
if err == nil {
63+
return nil
64+
}
65+
if apierrors.IsNotFound(err) {
66+
return fmt.Errorf("GPU device attachment requires DeviceClass %q", kvbuilder.GPUDeviceClassName)
67+
}
68+
return fmt.Errorf("failed to get GPU DeviceClass %q: %w", kvbuilder.GPUDeviceClassName, err)
69+
}

0 commit comments

Comments
 (0)