OCPBUGS-112537: E2E: Fix ovs dynamic pinning, kubelet and mustgather tests #1565
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
Contributor
There was a problem hiding this comment. The commit signature already contains AIA, we don't need the:
Contributor
Author
There was a problem hiding this comment. fixed in latest commit |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,7 +2,6 @@ package __performance_kubelet_node_test | |
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
|
|
@@ -24,6 +23,7 @@ import ( | |
| testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" | ||
| testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/infrastructure" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" | ||
| "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/poolname" | ||
|
|
@@ -41,14 +41,13 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| ) | ||
|
|
||
| testutils.CustomBeforeAll(func() { | ||
| var err error | ||
| workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| Expect(workerRTNodes).ToNot(BeEmpty(), "no RT worker nodes found") | ||
|
|
||
| profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
@@ -58,6 +57,12 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
|
|
||
| }) | ||
| Context("Additional kubelet arguments", Label(string(label.Tier2)), func() { | ||
| BeforeEach(func() { | ||
| var err error | ||
| profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| }) | ||
|
|
||
| It("[test_id:45488]Test performance profile annotation for changing multiple kubelet settings", func() { | ||
| sysctls := "{\"allowedUnsafeSysctls\":[\"net.core.somaxconn\",\"kernel.msg*\"],\"systemReserved\":{\"memory\":\"300Mi\"},\"kubeReserved\":{\"memory\":\"768Mi\"},\"imageMinimumGCAge\":\"3m\"}" | ||
| profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, sysctls) | ||
|
|
@@ -72,24 +77,25 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| profilesupdate.WaitForTuningUpdated(ctx, profile) | ||
|
|
||
| for _, node := range workerRTNodes { | ||
| kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| sysctlsValue := kubeletConfig.AllowedUnsafeSysctls | ||
| Expect(sysctlsValue).Should(ContainElements("net.core.somaxconn", "kernel.msg*")) | ||
| Expect(kubeletConfig.KubeReserved["memory"]).To(Equal("768Mi")) | ||
| Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).To(BeNumerically("==", 180)) | ||
| } | ||
|
|
||
| autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"} | ||
| for _, node := range workerRTNodes { | ||
| out, err := nodes.ExecCommand(ctx, &node, autoSizingCmd) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| stdout := testutils.ToString(out) | ||
| Expect(stdout).To(ContainSubstring("300Mi")) | ||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| }) | ||
| Context("When setting cpu manager related parameters", func() { | ||
| It("[test_id:45493]Should not override performance-addon-operator values", func() { | ||
| paoValues := "{\"cpuManagerPolicy\":\"none\",\"cpuManagerReconcilePeriod\":\"10s\"}" | ||
| profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, paoValues) | ||
|
|
||
| By("updating Performance profile") | ||
|
|
@@ -102,10 +108,10 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| profilesupdate.WaitForTuningUpdated(ctx, profile) | ||
|
|
||
| for _, node := range workerRTNodes { | ||
| kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(kubeletConfig.CPUManagerPolicy).Should(Equal("static")) | ||
| Expect(kubeletConfig.CPUManagerReconcilePeriod.Seconds()).To(BeNumerically("==", 5)) | ||
| } | ||
| }) | ||
| }) | ||
|
|
@@ -133,7 +139,7 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| Name: components.GetComponentName(profile.Name, components.ComponentNamePrefix), | ||
| Namespace: metav1.NamespaceNone, | ||
| } | ||
| err := testclient.ControlPlaneClient.Get(ctx, configKey, &kubeletConfig) | ||
| if err != nil { | ||
| klog.Warningf("Failed to get the KubeletConfig %q", configKey.Name) | ||
| } | ||
|
|
@@ -144,10 +150,19 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| Expect(kubeletConfigString).To(ContainSubstring(`"kubeReserved":{"memory":"768Mi"}`)) | ||
| Expect(kubeletConfigString).To(ContainSubstring(`"systemReserved":{"memory":"300Mi"}`)) | ||
|
|
||
| // Re-fetch nodes to get current allocatable and capacity after | ||
| // the tuning update, since workerRTNodes was populated before the | ||
| // annotation was applied and its Status values are stale. | ||
| updatedNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| updatedNodes, err = nodes.MatchingOptionalSelector(updatedNodes) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(updatedNodes).ToNot(BeEmpty(), "no RT worker nodes found after update") | ||
|
|
||
| for _, node := range updatedNodes { | ||
| kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| totalCapacity := node.Status.Capacity.Memory().MilliValue() | ||
| evictionMemory := kubeletConfig.EvictionHard["memory.available"] | ||
| kubeReserved := kubeletConfig.KubeReserved["memory"] | ||
| evictionMemoryInt, err := strconv.ParseInt(strings.TrimSuffix(evictionMemory, "Mi"), 10, 64) | ||
|
|
@@ -158,12 +173,32 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| kubeReservedMemoryResource := resource.NewQuantity(kubeReservedMemoryInt*1024*1024, resource.BinarySI) | ||
| evictionMemoryResource := resource.NewQuantity(evictionMemoryInt*1024*1024, resource.BinarySI) | ||
| totalKubeMemory := systemReservedResource.MilliValue() + kubeReservedMemoryResource.MilliValue() + evictionMemoryResource.MilliValue() | ||
|
|
||
| // Pre-allocated hugepages are subtracted from allocatable memory by the | ||
|
Contributor
There was a problem hiding this comment. There is no way to reserve hugepages on kubelet:
Contributor
Author
There was a problem hiding this comment. We are not reserving hugepages through kubelet, Hugepages are being set using Kernel from the parameters defined in Performance profile.
Contributor
There was a problem hiding this comment. got it, it's just a way to tell the kernel to make those HP so other applications can consume those later. |
||
| // kubelet but are still included in node capacity. The standard formula | ||
| // Allocatable = Capacity - kubeReserved - systemReserved - evictionHard | ||
| // does not account for this, so we must subtract hugepages to match the | ||
| // actual allocatable reported by the node. | ||
| var totalHugepages int64 | ||
| for resourceName, quantity := range node.Status.Capacity { | ||
| if strings.HasPrefix(string(resourceName), corev1.ResourceHugePagesPrefix) { | ||
| totalHugepages += quantity.MilliValue() | ||
| } | ||
| } | ||
|
|
||
| calculatedAllocatable := totalCapacity - totalKubeMemory - totalHugepages | ||
| currentAllocatable := node.Status.Allocatable.Memory().MilliValue() | ||
| Expect(calculatedAllocatable).To(Equal(currentAllocatable)) | ||
| } | ||
| }) | ||
|
|
||
| It("[test_id:45495] Test setting PAO managed parameters", func() { | ||
| isArm, err := infrastructure.IsARM(ctx, &workerRTNodes[0]) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| if isArm { | ||
| Skip("Changing topologyManagerPolicy is not supported on ARM architecture") | ||
| } | ||
|
|
||
| var paoParameters string | ||
| if *profile.Spec.NUMA.TopologyPolicy == "single-numa-node" { | ||
| paoParameters = "{\"topologyManagerPolicy\":\"restricted\"}" | ||
|
|
@@ -186,7 +221,7 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| profilesupdate.WaitForTuningUpdated(ctx, profile) | ||
| } | ||
| for _, node := range workerRTNodes { | ||
| kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(kubeletConfig.TopologyManagerPolicy).To(Equal("single-numa-node")) | ||
| } | ||
|
|
@@ -196,38 +231,35 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord | |
| By("Reverting the Profile") | ||
| profiles.UpdateWithRetry(initialProfile) | ||
|
|
||
| By(fmt.Sprintf("Waiting for %s to be updated after revert", poolName)) | ||
| profilesupdate.WaitForTuningUpdated(ctx, initialProfile) | ||
|
|
||
| for _, node := range workerRTNodes { | ||
| kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| Expect(kubeletConfig.AllowedUnsafeSysctls).To(BeEmpty()) | ||
| Expect(kubeletConfig.KubeReserved["memory"]).ToNot(Equal("768Mi")) | ||
| Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).ToNot(Equal(180)) | ||
| } | ||
|
|
||
| autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"} | ||
| for _, node := range workerRTNodes { | ||
| out, err := nodes.ExecCommand(ctx, &node, autoSizingCmd) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| stdout := testutils.ToString(out) | ||
| Expect(stdout).ToNot(ContainSubstring("300Mi")) | ||
| } | ||
|
|
||
| }) | ||
| AfterAll(func() { | ||
| By("Reverting the Profile") | ||
| profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| if !equality.Semantic.DeepEqual(profile.Spec, initialProfile.Spec) || !equality.Semantic.DeepEqual(profile.Annotations, initialProfile.Annotations) { | ||
| profiles.UpdateWithRetry(initialProfile) | ||
|
|
||
| By(fmt.Sprintf("Waiting for %s to be updated after revert", poolName)) | ||
| profilesupdate.WaitForTuningUpdated(ctx, initialProfile) | ||
| } | ||
| }) | ||
|
|
||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.