Skip to content

add repoconfig to the openshift-driver-toolkit-ctr container in DTK mode #2812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kvalliyurnatt wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add repoconfig to the openshift-driver-toolkit-ctr container in DTK mode #2812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions controllers/object_controls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3716,6 +3716,13 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom repo config: %v", err)
}
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...)
// Mount the repo config volume to the openshift-driver-toolkit container if it is enabled
if n.ocpDriverToolkit.enabled {
driverToolkitContainer := findContainerByName(podSpec.Containers, "openshift-driver-toolkit-ctr")
if driverToolkitContainer != nil {
driverToolkitContainer.VolumeMounts = append(driverToolkitContainer.VolumeMounts, volumeMounts...)
}
}
podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.RepoConfig.ConfigMapName, itemsToInclude))
}

Expand All @@ -3730,6 +3737,12 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom certs: %w", err)
}
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...)
if n.ocpDriverToolkit.enabled {
driverToolkitContainer := findContainerByName(podSpec.Containers, "openshift-driver-toolkit-ctr")
if driverToolkitContainer != nil {
driverToolkitContainer.VolumeMounts = append(driverToolkitContainer.VolumeMounts, volumeMounts...)
}
}
podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.CertConfig.Name, itemsToInclude))
}

Expand Down
78 changes: 78 additions & 0 deletions controllers/transforms_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4652,6 +4652,84 @@ func TestTransformDriverWithAdditionalConfig(t *testing.T) {
}
}

func TestTransformDriverMountsCustomConfigsInDriverToolkit(t *testing.T) {
repoConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo-config",
Namespace: "test-ns",
},
Data: map[string]string{
"custom.repo": "[custom-repo]",
},
}
certConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cert-config",
Namespace: "test-ns",
},
Data: map[string]string{
"custom.pem": "test-certificate",
},
}

ds := NewDaemonset().
WithContainer(corev1.Container{Name: "nvidia-driver-ctr"}).
WithContainer(corev1.Container{Name: "openshift-driver-toolkit-ctr"})
config := &gpuv1.ClusterPolicySpec{
Driver: gpuv1.DriverSpec{
Repository: "nvcr.io/nvidia",
Image: "driver",
Version: "580.126.16",
ImagePullPolicy: "IfNotPresent",
RepoConfig: &gpuv1.DriverRepoConfigSpec{
ConfigMapName: repoConfigMap.Name,
},
CertConfig: &gpuv1.DriverCertConfigSpec{
Name: certConfigMap.Name,
},
},
}
controller := ClusterPolicyController{
client: fake.NewFakeClient(repoConfigMap, certConfigMap),
operatorNamespace: "test-ns",
gpuNodeOSRelease: "rhcos",
gpuNodeOSTag: "rhcos4.22",
ocpDriverToolkit: OpenShiftDriverToolkit{
enabled: true,
},
}

require.NoError(t, transformDriverContainer(ds.DaemonSet, config, controller))

driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
require.NotNil(t, driverContainer)
driverToolkitContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "openshift-driver-toolkit-ctr")
require.NotNil(t, driverToolkitContainer)

expectedMounts := []corev1.VolumeMount{
{
Name: repoConfigMap.Name,
ReadOnly: true,
MountPath: "/etc/yum.repos.d/custom.repo",
SubPath: "custom.repo",
},
{
Name: certConfigMap.Name,
ReadOnly: true,
MountPath: "/etc/pki/ca-trust/extracted/pem/custom.pem",
SubPath: "custom.pem",
},
}
for _, expectedMount := range expectedMounts {
assert.Contains(t, driverContainer.VolumeMounts, expectedMount)
assert.Contains(t, driverToolkitContainer.VolumeMounts, expectedMount)
}

require.Len(t, ds.Spec.Template.Spec.Volumes, 2)
assert.Equal(t, repoConfigMap.Name, ds.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, certConfigMap.Name, ds.Spec.Template.Spec.Volumes[1].Name)
}

func TestTransformDriverSubscriptionMounts(t *testing.T) {
repoConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Expand Down
5 changes: 3 additions & 2 deletions internal/state/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,9 @@ type precompiledSpec struct {
}

type additionalConfigs struct {
VolumeMounts []corev1.VolumeMount
Volumes []corev1.Volume
VolumeMounts []corev1.VolumeMount
DriverToolkitVolumeMounts []corev1.VolumeMount
Volumes []corev1.Volume
}

type driverRenderData struct {
Expand Down
87 changes: 87 additions & 0 deletions internal/state/driver_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,93 @@ func TestDriverOpenshiftDriverToolkit(t *testing.T) {
require.Equal(t, string(o), actual)
}

func TestDriverOpenshiftDriverToolkitCustomConfigs(t *testing.T) {
const (
rhcosVersion = "422.92.202608252344-0"
toolkitImage = "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:test"
)

repoConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-repo-config",
Namespace: "test-ns",
},
Data: map[string]string{
"custom.repo": "[custom-repo]",
},
}
certConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cert-config",
Namespace: "test-ns",
},
Data: map[string]string{
"custom.pem": "test-certificate",
},
}

state, err := NewStateDriver(
fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(repoConfigMap, certConfigMap).Build(),
"test-ns",
scheme.Scheme,
manifestDir)
require.NoError(t, err)
stateDriver, ok := state.(*stateDriver)
require.True(t, ok)

driver := &nvidiav1alpha1.NVIDIADriver{
Spec: nvidiav1alpha1.NVIDIADriverSpec{
RepoConfig: &nvidiav1alpha1.DriverRepoConfigSpec{Name: repoConfigMap.Name},
CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: certConfigMap.Name},
},
}
additionalConfigs, err := stateDriver.getDriverAdditionalConfigs(
context.Background(),
driver,
testClusterInfo{runtime: consts.CRIO, openshiftVersion: "4.22"},
nodePool{osRelease: "rhcos", osVersion: "4.22"},
)
require.NoError(t, err)

renderData := getMinimalDriverRenderData()
renderData.Driver.Spec.RepoConfig = driver.Spec.RepoConfig
renderData.Driver.Spec.CertConfig = driver.Spec.CertConfig
renderData.AdditionalConfigs = additionalConfigs
renderData.Openshift = &openshiftSpec{
ToolkitImage: toolkitImage,
RHCOSVersion: rhcosVersion,
}
renderData.Runtime.OpenshiftDriverToolkitEnabled = true
renderData.Runtime.OpenshiftVersion = "4.22"

objs, err := stateDriver.renderer.RenderObjects(
&render.TemplatingData{
Data: renderData,
})
require.NoError(t, err)

ds, err := getDaemonsetFromObjects(objs)
require.NoError(t, err)

driverToolkitContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "openshift-driver-toolkit-ctr")
require.NotNil(t, driverToolkitContainer)

repoMount := findVolumeMountByName(driverToolkitContainer.VolumeMounts, repoConfigMap.Name)
require.NotNil(t, repoMount)
assert.Equal(t, "/etc/yum.repos.d/custom.repo", repoMount.MountPath)
assert.Equal(t, "custom.repo", repoMount.SubPath)
assert.True(t, repoMount.ReadOnly)

certMount := findVolumeMountByName(driverToolkitContainer.VolumeMounts, certConfigMap.Name)
require.NotNil(t, certMount)
assert.Equal(t, "/etc/pki/ca-trust/extracted/pem/custom.pem", certMount.MountPath)
assert.Equal(t, "custom.pem", certMount.SubPath)
assert.True(t, certMount.ReadOnly)

assert.NotNil(t, findVolumeByName(ds.Spec.Template.Spec.Volumes, repoConfigMap.Name))
assert.NotNil(t, findVolumeByName(ds.Spec.Template.Spec.Volumes, certConfigMap.Name))
}

func TestGetNodePoolsDoesNotAllowSelectorToOverrideOwnerLabel(t *testing.T) {
require.NoError(t, corev1.AddToScheme(scheme.Scheme))

Expand Down
2 changes: 2 additions & 0 deletions internal/state/driver_volumes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph
return nil, fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom repo config: %w", err)
}
additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volumeMounts...)
additionalCfgs.DriverToolkitVolumeMounts = append(additionalCfgs.DriverToolkitVolumeMounts, volumeMounts...)
additionalCfgs.Volumes = append(additionalCfgs.Volumes, createConfigMapVolume(cr.Spec.RepoConfig.Name, itemsToInclude))
}

Expand All @@ -169,6 +170,7 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph
return nil, fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom certs: %w", err)
}
additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, volumeMounts...)
additionalCfgs.DriverToolkitVolumeMounts = append(additionalCfgs.DriverToolkitVolumeMounts, volumeMounts...)
additionalCfgs.Volumes = append(additionalCfgs.Volumes, createConfigMapVolume(cr.Spec.CertConfig.Name, itemsToInclude))
}

Expand Down
12 changes: 12 additions & 0 deletions manifests/state-driver/0500_daemonset.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,18 @@ spec:
mountPath: /sys/module/firmware_class/parameters/path
- name: nv-firmware
mountPath: /lib/firmware
{{- if and .AdditionalConfigs .AdditionalConfigs.DriverToolkitVolumeMounts }}
{{- range .AdditionalConfigs.DriverToolkitVolumeMounts }}
- name: {{ .Name }}
mountPath: {{ .MountPath }}
{{- if .SubPath }}
subPath: {{ .SubPath }}
{{- end }}
{{- if .ReadOnly }}
readOnly: {{ .ReadOnly }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Driver.Spec.Resources }}
resources: {{ .Driver.Spec.Resources | yaml | nindent 10 }}
{{- end }}
Expand Down
Loading