diff --git a/controllers/object_controls.go b/controllers/object_controls.go index 0113a43b6..0b19384b6 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -3716,6 +3716,13 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom repo config: %v", err) } driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...) + // Mount the repo config volume to the openshift-driver-toolkit container if it is enabled + if n.ocpDriverToolkit.enabled { + driverToolkitContainer := findContainerByName(podSpec.Containers, "openshift-driver-toolkit-ctr") + if driverToolkitContainer != nil { + driverToolkitContainer.VolumeMounts = append(driverToolkitContainer.VolumeMounts, volumeMounts...) + } + } podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.RepoConfig.ConfigMapName, itemsToInclude)) } @@ -3730,6 +3737,12 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom certs: %w", err) } driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...) + if n.ocpDriverToolkit.enabled { + driverToolkitContainer := findContainerByName(podSpec.Containers, "openshift-driver-toolkit-ctr") + if driverToolkitContainer != nil { + driverToolkitContainer.VolumeMounts = append(driverToolkitContainer.VolumeMounts, volumeMounts...) + } + } podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.CertConfig.Name, itemsToInclude)) } diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index aa309a322..a8e93e7be 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -4652,6 +4652,84 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) { } } +func TestTransformDriverMountsCustomConfigsInDriverToolkit(t *testing.T) { + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.repo": "[custom-repo]", + }, + } + certConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cert-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.pem": "test-certificate", + }, + } + + ds := NewDaemonset(). + WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}). + WithContainer(corev1.Container{Name: "openshift-driver-toolkit-ctr"}) + config := &gpuv1.ClusterPolicySpec{ + Driver: gpuv1.DriverSpec{ + Repository: "nvcr.io/nvidia", + Image: "driver", + Version: "580.126.16", + ImagePullPolicy: "IfNotPresent", + RepoConfig: &gpuv1.DriverRepoConfigSpec{ + ConfigMapName: repoConfigMap.Name, + }, + CertConfig: &gpuv1.DriverCertConfigSpec{ + Name: certConfigMap.Name, + }, + }, + } + controller := ClusterPolicyController{ + client: fake.NewFakeClient(repoConfigMap, certConfigMap), + operatorNamespace: "test-ns", + gpuNodeOSRelease: "rhcos", + gpuNodeOSTag: "rhcos4.22", + ocpDriverToolkit: OpenShiftDriverToolkit{ + enabled: true, + }, + } + + require.NoError(t, transformDriverContainer(ds.DaemonSet, config, controller)) + + driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr") + require.NotNil(t, driverContainer) + driverToolkitContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "openshift-driver-toolkit-ctr") + require.NotNil(t, driverToolkitContainer) + + expectedMounts := []corev1.VolumeMount{ + { + Name: repoConfigMap.Name, + ReadOnly: true, + MountPath: "/etc/yum.repos.d/custom.repo", + SubPath: "custom.repo", + }, + { + Name: certConfigMap.Name, + ReadOnly: true, + MountPath: "/etc/pki/ca-trust/extracted/pem/custom.pem", + SubPath: "custom.pem", + }, + } + for _, expectedMount := range expectedMounts { + assert.Contains(t, driverContainer.VolumeMounts, expectedMount) + assert.Contains(t, driverToolkitContainer.VolumeMounts, expectedMount) + } + + require.Len(t, ds.Spec.Template.Spec.Volumes, 2) + assert.Equal(t, repoConfigMap.Name, ds.Spec.Template.Spec.Volumes[0].Name) + assert.Equal(t, certConfigMap.Name, ds.Spec.Template.Spec.Volumes[1].Name) +} + func TestTransformDriverSubscriptionMounts(t *testing.T) { repoConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/state/driver.go b/internal/state/driver.go index 2046bbf81..788d280dd 100644 --- a/internal/state/driver.go +++ b/internal/state/driver.go @@ -84,8 +84,9 @@ type precompiledSpec struct { } type additionalConfigs struct { - VolumeMounts []corev1.VolumeMount - Volumes []corev1.Volume + VolumeMounts []corev1.VolumeMount + DriverToolkitVolumeMounts []corev1.VolumeMount + Volumes []corev1.Volume } type driverRenderData struct { diff --git a/internal/state/driver_test.go b/internal/state/driver_test.go index 92907fb62..c5c640254 100644 --- a/internal/state/driver_test.go +++ b/internal/state/driver_test.go @@ -869,6 +869,93 @@ func TestDriverOpenshiftDriverToolkit(t *testing.T) { require.Equal(t, string(o), actual) } +func TestDriverOpenshiftDriverToolkitCustomConfigs(t *testing.T) { + const ( + rhcosVersion = "422.92.202608252344-0" + toolkitImage = "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:test" + ) + + repoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-repo-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.repo": "[custom-repo]", + }, + } + certConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cert-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "custom.pem": "test-certificate", + }, + } + + state, err := NewStateDriver( + fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap, certConfigMap).Build(), + "test-ns", + scheme.Scheme, + manifestDir) + require.NoError(t, err) + stateDriver, ok := state.(*stateDriver) + require.True(t, ok) + + driver := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + RepoConfig: &nvidiav1alpha1.DriverRepoConfigSpec{Name: repoConfigMap.Name}, + CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: certConfigMap.Name}, + }, + } + additionalConfigs, err := stateDriver.getDriverAdditionalConfigs( + context.Background(), + driver, + testClusterInfo{runtime: consts.CRIO, openshiftVersion: "4.22"}, + nodePool{osRelease: "rhcos", osVersion: "4.22"}, + ) + require.NoError(t, err) + + renderData := getMinimalDriverRenderData() + renderData.Driver.Spec.RepoConfig = driver.Spec.RepoConfig + renderData.Driver.Spec.CertConfig = driver.Spec.CertConfig + renderData.AdditionalConfigs = additionalConfigs + renderData.Openshift = &openshiftSpec{ + ToolkitImage: toolkitImage, + RHCOSVersion: rhcosVersion, + } + renderData.Runtime.OpenshiftDriverToolkitEnabled = true + renderData.Runtime.OpenshiftVersion = "4.22" + + objs, err := stateDriver.renderer.RenderObjects( + &render.TemplatingData{ + Data: renderData, + }) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + driverToolkitContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "openshift-driver-toolkit-ctr") + require.NotNil(t, driverToolkitContainer) + + repoMount := findVolumeMountByName(driverToolkitContainer.VolumeMounts, repoConfigMap.Name) + require.NotNil(t, repoMount) + assert.Equal(t, "/etc/yum.repos.d/custom.repo", repoMount.MountPath) + assert.Equal(t, "custom.repo", repoMount.SubPath) + assert.True(t, repoMount.ReadOnly) + + certMount := findVolumeMountByName(driverToolkitContainer.VolumeMounts, certConfigMap.Name) + require.NotNil(t, certMount) + assert.Equal(t, "/etc/pki/ca-trust/extracted/pem/custom.pem", certMount.MountPath) + assert.Equal(t, "custom.pem", certMount.SubPath) + assert.True(t, certMount.ReadOnly) + + assert.NotNil(t, findVolumeByName(ds.Spec.Template.Spec.Volumes, repoConfigMap.Name)) + assert.NotNil(t, findVolumeByName(ds.Spec.Template.Spec.Volumes, certConfigMap.Name)) +} + func TestGetNodePoolsDoesNotAllowSelectorToOverrideOwnerLabel(t *testing.T) { require.NoError(t, corev1.AddToScheme(scheme.Scheme)) diff --git a/internal/state/driver_volumes.go b/internal/state/driver_volumes.go index 34f60ea60..d4b713e4e 100644 --- a/internal/state/driver_volumes.go +++ b/internal/state/driver_volumes.go @@ -154,6 +154,7 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph return nil, fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom repo config: %w", err) } additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volumeMounts...) + additionalCfgs.DriverToolkitVolumeMounts = append(additionalCfgs.DriverToolkitVolumeMounts, volumeMounts...) additionalCfgs.Volumes = append(additionalCfgs.Volumes, createConfigMapVolume(cr.Spec.RepoConfig.Name, itemsToInclude)) } @@ -169,6 +170,7 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph return nil, fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom certs: %w", err) } additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volumeMounts...) + additionalCfgs.DriverToolkitVolumeMounts = append(additionalCfgs.DriverToolkitVolumeMounts, volumeMounts...) additionalCfgs.Volumes = append(additionalCfgs.Volumes, createConfigMapVolume(cr.Spec.CertConfig.Name, itemsToInclude)) } diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml index 65e1eb74a..af25caf42 100644 --- a/manifests/state-driver/0500_daemonset.yaml +++ b/manifests/state-driver/0500_daemonset.yaml @@ -630,6 +630,18 @@ spec: mountPath: /sys/module/firmware_class/parameters/path - name: nv-firmware mountPath: /lib/firmware + {{- if and .AdditionalConfigs .AdditionalConfigs.DriverToolkitVolumeMounts }} + {{- range .AdditionalConfigs.DriverToolkitVolumeMounts }} + - name: {{ .Name }} + mountPath: {{ .MountPath }} + {{- if .SubPath }} + subPath: {{ .SubPath }} + {{- end }} + {{- if .ReadOnly }} + readOnly: {{ .ReadOnly }} + {{- end }} + {{- end }} + {{- end }} {{- if .Driver.Spec.Resources }} resources: {{ .Driver.Spec.Resources | yaml | nindent 10 }} {{- end }}