feat(ci): run e2e tests from the runner via Holodeck remoteAccess

.github/workflows/e2e-tests.yaml

@@ -55,6 +55,12 @@ on:
         type: boolean
         default: false
 
+env:
+  HELM_VERSION: v3.19.0
+  KUBECTL_VERSION: v1.35.4
+  JQ_VERSION: "1.7.1"
+  JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"
+
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
@@ -79,6 +85,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -92,11 +101,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -109,45 +136,73 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/defaults.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/defaults.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -156,6 +211,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -169,11 +227,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -186,42 +262,70 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/nvidia-driver.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/nvidia-driver.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"

tests/README.md

@@ -1,5 +1,22 @@
 # GPU operator test utilities
 
+## Testing in CI
+CI no longer uses `local.sh` or `ci-run-e2e.sh`, and no longer syncs the project
+folder to the test instance. Those remain the developer path described below.
+
+Instead, the e2e workflow provisions a Holodeck environment with
+`kubernetes.remoteAccess` enabled, which gives the GitHub Actions runner a
+kubeconfig for the cluster. The case scripts (`cases/defaults.sh`,
+`cases/nvidia-driver.sh`) then run on the runner itself, and everything they do
+-- helm, kubectl, log collection -- goes over that kubeconfig.
+
+Only two operations still need a shell on the instance, and both go through
+`scripts/node-exec.sh`, which dispatches `scripts/node-operations.sh` over SSH:
+loading the `i2c_core` and `ipmi_msghandler` kernel modules, and killing the
+gpu-operator container for the operator restart test. With `NODE_SSH_HOST`
+unset, `node-exec.sh` runs the operation locally, so the developer path below is
+unaffected.
+
 ## Testing locally
 The `local.sh` script allows for triggering basic end-to-end testing of the GPU
 operator from a local machine.

tests/holodeck.yaml

@@ -24,3 +24,4 @@ spec:
     version: v1.35.4
     crictlVersion: v1.35.0
     calicoVersion: v3.31.5
+    remoteAccess: true

tests/scripts/.definitions.sh

@@ -14,6 +14,8 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 
 # Set default values if not defined
 : ${HELM:="helm"}
+: "${KUBECTL_REQUEST_TIMEOUT:="15s"}"
+: "${KUBECTL_LOG_TIMEOUT:="120s"}"
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}