From 94cc00b39ff597ddc5b58ab52de96ded72322e28 Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 31 Aug 2026 14:52:20 +0200 Subject: [PATCH 1/8] util: implement GetSecret for the KMIP provider fscrypt needs one deterministic secret from the KMS and builds its own key hierarchy on the volume, so the stub that always answered ErrGetSecretUnsupported was the only thing keeping a KMIP KMS from backing CephFS file encryption and RBD encryptionType file. Return the key material of the managed symmetric key, base64 encoded so a volume stays openable with the fscrypt tool, and keep rejecting the request when USE_CRYPTO_RPC keeps cryptographic operations on the server. Assisted-by: Claude Code Signed-off-by: David Mohren --- internal/kms/kmip.go | 16 +++++++- internal/kms/kmip_test.go | 85 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/internal/kms/kmip.go b/internal/kms/kmip.go index d93acaea2fb..099cb6ca758 100644 --- a/internal/kms/kmip.go +++ b/internal/kms/kmip.go @@ -24,6 +24,7 @@ import ( "crypto/cipher" "crypto/tls" "crypto/x509" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -265,8 +266,19 @@ func (kms *kmipKMS) RequiresDEKStore() DEKStoreType { return DEKStoreMetadata } -func (kms *kmipKMS) GetSecret(ctx context.Context, volumeID string) (string, error) { - return "", ErrGetSecretUnsupported +// GetSecret gets the raw secret via KMIP. +func (kms *kmipKMS) GetSecret(_ context.Context, _ string) (string, error) { + if kms.useCryptoRPC { + return "", fmt.Errorf("%w: fscrypt requires the key material, set %q to false", + ErrGetSecretUnsupported, kmipUseCryptoRPC) + } + + key, err := kms.getKey(kms.uniqueIdentifier) + if err != nil { + return "", fmt.Errorf("failed to get key %q: %w", kms.uniqueIdentifier, err) + } + + return base64.StdEncoding.EncodeToString(key), nil } // encryptDEKUsingEncryptRPC uses the KMIP encrypt operation to encrypt the DEK. diff --git a/internal/kms/kmip_test.go b/internal/kms/kmip_test.go index 29d212d3cb2..d88a542f414 100644 --- a/internal/kms/kmip_test.go +++ b/internal/kms/kmip_test.go @@ -17,12 +17,15 @@ limitations under the License. package kms import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" + "errors" "math/big" "net" "testing" @@ -97,7 +100,7 @@ func TestKMIPConnectTLSMinVersion(t *testing.T) { certificate, caCertPool := kmipTestCertificate(t) kms := &kmipKMS{ - endpoint: kmipTestServer(t, &certificate, test.serverMax), + endpoint: kmipTestServer(t, &certificate, test.serverMax, nil), tlsConfig: &tls.Config{ MinVersion: test.clientMin, RootCAs: caCertPool, @@ -122,10 +125,59 @@ func TestKMIPConnectTLSMinVersion(t *testing.T) { } } +func TestKMIPGetSecretUnsupportedWithCryptoRPC(t *testing.T) { + t.Parallel() + + kms := &kmipKMS{ + useCryptoRPC: true, + } + + _, err := kms.GetSecret(context.TODO(), "") + require.ErrorIs(t, err, ErrGetSecretUnsupported) + require.ErrorContains(t, err, kmipUseCryptoRPC) +} + +func TestKMIPGetSecret(t *testing.T) { + t.Parallel() + + keyUID := "kmip-test-key-uid" + keyMaterial := make([]byte, 32) + _, err := rand.Read(keyMaterial) + require.NoError(t, err) + + certificate, caCertPool := kmipTestCertificate(t) + + kms := &kmipKMS{ + endpoint: kmipTestServer(t, &certificate, tls.VersionTLS13, map[string][]byte{keyUID: keyMaterial}), + tlsConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caCertPool, + Certificates: []tls.Certificate{certificate}, + }, + uniqueIdentifier: keyUID, + readTimeout: kmipDefaulfReadTimeout, + writeTimeout: kmipDefaultWriteTimeout, + useCryptoRPC: false, + } + + secret, err := kms.GetSecret(context.TODO(), "") + require.NoError(t, err) + require.Equal(t, base64.StdEncoding.EncodeToString(keyMaterial), secret) + + // the passphrase has to be reproducible for the lifetime of the + // volume, a second call must return the identical value + again, err := kms.GetSecret(context.TODO(), "") + require.NoError(t, err) + require.Equal(t, secret, again) +} + // kmipTestServer runs an in-process KMIP server behind TLS, offering no more // than the given TLS version, and returns its endpoint. It answers the -// DiscoverVersions exchange that connect() performs, and nothing else. -func kmipTestServer(t *testing.T, certificate *tls.Certificate, maxVersion uint16) string { +// DiscoverVersions exchange that connect() performs, and serves the symmetric +// keys in keys through the Get operation. +func kmipTestServer( + t *testing.T, certificate *tls.Certificate, maxVersion uint16, keys map[string][]byte, +) string { t.Helper() version := kmip.ProtocolVersion{ @@ -138,6 +190,33 @@ func kmipTestServer(t *testing.T, certificate *tls.Certificate, maxVersion uint1 SupportedVersions: []kmip.ProtocolVersion{version}, }) + if keys != nil { + mux.Handle(kmip14.OperationGet, &kmip.GetHandler{ + Get: func(_ context.Context, payload *kmip.GetRequestPayload) (*kmip.GetResponsePayload, error) { + keyMaterial, ok := keys[payload.UniqueIdentifier] + if !ok { + return nil, kmip.WithResultReason( + errors.New("no such key"), kmip14.ResultReasonItemNotFound) + } + + return &kmip.GetResponsePayload{ + ObjectType: kmip14.ObjectTypeSymmetricKey, + UniqueIdentifier: payload.UniqueIdentifier, + SymmetricKey: &kmip.SymmetricKey{ + KeyBlock: kmip.KeyBlock{ + KeyFormatType: kmip14.KeyFormatTypeRaw, + KeyValue: &kmip.KeyValue{ + KeyMaterial: keyMaterial, + }, + CryptographicAlgorithm: kmip14.CryptographicAlgorithmAES, + CryptographicLength: len(keyMaterial) * 8, + }, + }, + }, nil + }, + }) + } + server := &kmip.Server{ Handler: &kmip.StandardProtocolHandler{ ProtocolVersion: version, From f05394c009959e309543ccb7db3eb84a0e2c8d1d Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 31 Aug 2026 17:56:11 +0200 Subject: [PATCH 2/8] rbd: reject fscrypt file encryption with the kmip KMS Implementing GetSecret on the kmip provider makes it pass the fscrypt capability probe for RBD encryptionType file as well, but only the CephFS combination has been tested. Keep RBD failing at CreateVolume, as it does today, with an actionable error instead of the accidental probe rejection. The lock is removed once the combination is validated. Assisted-by: Claude Code Signed-off-by: David Mohren --- internal/kms/kmip.go | 6 ++++++ internal/kms/kmip_test.go | 6 ++++++ internal/rbd/encryption.go | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/internal/kms/kmip.go b/internal/kms/kmip.go index 099cb6ca758..48112437d36 100644 --- a/internal/kms/kmip.go +++ b/internal/kms/kmip.go @@ -281,6 +281,12 @@ func (kms *kmipKMS) GetSecret(_ context.Context, _ string) (string, error) { return base64.StdEncoding.EncodeToString(key), nil } +func IsKMIP(ekms EncryptionKMS) bool { + _, ok := ekms.(*kmipKMS) + + return ok +} + // encryptDEKUsingEncryptRPC uses the KMIP encrypt operation to encrypt the DEK. func (kms *kmipKMS) encryptDEKUsingEncryptRPC(_ context.Context, _, plainDEK string) (string, error) { conn, err := kms.connect() diff --git a/internal/kms/kmip_test.go b/internal/kms/kmip_test.go index d88a542f414..92a49089ec0 100644 --- a/internal/kms/kmip_test.go +++ b/internal/kms/kmip_test.go @@ -125,6 +125,12 @@ func TestKMIPConnectTLSMinVersion(t *testing.T) { } } +func TestIsKMIP(t *testing.T) { + t.Parallel() + require.True(t, IsKMIP(&kmipKMS{})) + require.False(t, IsKMIP(secretsMetadataKMS{})) +} + func TestKMIPGetSecretUnsupportedWithCryptoRPC(t *testing.T) { t.Parallel() diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index ffbc7596625..949415624c1 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -464,6 +464,14 @@ func (ri *rbdImage) configureFileEncryption(ctx context.Context, kmsID string, c if err != nil { return err } + + // the kmip KMS answers the GetSecret capability probe below, but the + // combination with fscrypt on RBD has not been tested yet + if kmsapi.IsKMIP(kms) { + return fmt.Errorf("KMS %q uses the kmip provider, which is not tested "+ + "with fscrypt file encryption on RBD", kmsID) + } + // Not usesable for filestorage encryption ri.fileEncryption, err = util.NewVolumeEncryption(kmsID, kms, nil) From 0c66413da733cead7d49d22cecd978ca63f7c997 Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 31 Aug 2026 14:52:28 +0200 Subject: [PATCH 3/8] doc: announce fscrypt encryption with a KMIP KMS Document that a KMIP KMS backs CephFS fscrypt encryption when USE_CRYPTO_RPC is disabled, and warn that rotating or replacing the managed key makes existing volumes permanently unopenable. Assisted-by: Claude Code Signed-off-by: David Mohren --- PendingReleaseNotes.md | 5 +++++ docs/cephfs/deploy.md | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index b29299976ec..bebd47c680c 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -13,5 +13,10 @@ server that only offers TLS 1.2 instead of silently continuing over TLS 1.2. The negotiated TLS version and cipher suite are now logged for each KMIP connection. +1. CephFS: fscrypt file encryption now works with a KMIP KMS when + `USE_CRYPTO_RPC` is set to `"false"`. The key material of the managed + symmetric key is fetched with the KMIP `Get` operation and used as the + fscrypt passphrase. RBD with `encryptionType: file` keeps rejecting a + KMIP KMS until that combination has been tested. ## NOTE diff --git a/docs/cephfs/deploy.md b/docs/cephfs/deploy.md index 1be5bf6f022..f023a9c95f1 100644 --- a/docs/cephfs/deploy.md +++ b/docs/cephfs/deploy.md @@ -269,6 +269,13 @@ However, not all KMS are supported in order to be compatible with either store secrets to use directly (Vault), or allow access to the plain password (Kubernetes Secrets) work. +A KMIP KMS works when `USE_CRYPTO_RPC` is set to `"false"`, so that +Ceph-CSI can fetch the key material of the managed symmetric key with +the KMIP `Get` operation and use it as the fscrypt passphrase. Note +that rotating or destroying the KMIP key, or pointing +`UNIQUE_IDENTIFIER` at a different key, makes every existing encrypted +volume permanently unopenable. + ## CephFS PVC Provisioning Requires subvolumegroup to be created before provisioning the PVC. From 6b50457c48206c1e2a27190c458986ef593e0bca Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 31 Aug 2026 16:40:43 +0200 Subject: [PATCH 4/8] e2e: deploy a PyKMIP server for KMIP KMS testing No KMS speaking KMIP exists in the e2e environment, so the KMIP provider has never been covered there. Deploy PyKMIP, pinned together with the last dependency versions it works with, and provisions what the kmip KMS provider needs. PyKMIP identifies clients by the certificate CN and only lets the owner fetch a key, so the key-creating script uses the same client certificate as ceph-csi. PyKMIP 0.10.0 is the last release and predates current versions of its dependencies, so Python, SQLAlchemy and cryptography are pinned to the last versions it works with. Newer cryptography removed the legacy ciphers PyKMIP imports and the server crashes on startup. Assisted-by: Claude Code Signed-off-by: David Mohren --- e2e/README.md | 18 ++ e2e/deploy-kmip.go | 294 +++++++++++++++++++++++++++++++++ e2e/kms.go | 7 + examples/kms/vault/pykmip.yaml | 129 +++++++++++++++ 4 files changed, 448 insertions(+) create mode 100644 e2e/deploy-kmip.go create mode 100644 examples/kms/vault/pykmip.yaml diff --git a/e2e/README.md b/e2e/README.md index c14e8fa1226..03e69bdfb55 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -226,3 +226,21 @@ You can also invoke functional tests with `make` command ```console make func-test TESTOPTIONS="-deploy-timeout=10 -timeout=30m -v" ``` + +## KMS services for the encryption tests + +The encryption tests read the KMS configurations from +`examples/kms/vault/kms-config.yaml` and deploy the KMS services they +need into the ceph-csi namespace: + +- HashiCorp Vault is deployed for every run, unless `--skip-vault=true` + is set. +- A PyKMIP server is deployed in addition when + `--test-cephfs-fscrypt=true` is set, to serve the `kmip-fscrypt-test` + configuration. The e2e tests generate the TLS certificates, create an + AES key on the server and store its unique identifier in the + `ceph-csi-kmip-credentials` Secret, see `e2e/deploy-kmip.go`. + +The PyKMIP Pod installs PyKMIP at startup. Its version and dependency +pins, and the reasons for them, are documented in +`examples/kms/vault/pykmip.yaml`. diff --git a/e2e/deploy-kmip.go b/e2e/deploy-kmip.go new file mode 100644 index 00000000000..5704e5bcbf4 --- /dev/null +++ b/e2e/deploy-kmip.go @@ -0,0 +1,294 @@ +/* +Copyright 2026 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "strings" + "time" + + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + pykmipManifest = "pykmip.yaml" + kmipDeploymentName = "kmip" + + // kmipCertsSecretName is mounted by the PyKMIP Deployment and holds + // the TLS material for the server and the client. + // + // #nosec:G101, value not credential, name of a Kubernetes Secret. + kmipCertsSecretName = "ceph-csi-kmip-certs" + + // kmipCredentialsName must match the default KMIP_SECRET_NAME of the + // kmip KMS provider. + // + // #nosec:G101, value not credential, name of a Kubernetes Secret. + kmipCredentialsName = "ceph-csi-kmip-credentials" +) + +// kmipCertificates holds the PEM encoded TLS material for the PyKMIP server +// and its clients. PyKMIP identifies a client by the CN of its certificate +// and only lets the owner fetch a key, so the key-creating script and +// ceph-csi use the same client certificate. +type kmipCertificates struct { + caCert string + serverCert string + serverKey string + clientCert string + clientKey string +} + +// deployKMIP deploys a PyKMIP server and provisions everything the kmip KMS +// provider needs: the TLS certificates, an AES key on the server and the +// credentials Secret naming that key. +func deployKMIP(f *framework.Framework, deployTimeout int) { + certs, err := generateKMIPCertificates(cephCSINamespace) + Expect(err).ShouldNot(HaveOccurred()) + + err = createKMIPSecret(f, kmipCertsSecretName, map[string]string{ + "ca.crt": certs.caCert, + "server.crt": certs.serverCert, + "server.key": certs.serverKey, + "client.crt": certs.clientCert, + "client.key": certs.clientKey, + }) + Expect(err).ShouldNot(HaveOccurred()) + + data, err := replaceNamespaceInTemplate(vaultExamplePath + pykmipManifest) + if err != nil { + logAndFail("failed to read content from %s: %v", vaultExamplePath+pykmipManifest, err) + } + err = retryKubectlInput(cephCSINamespace, kubectlCreate, data, deployTimeout) + if err != nil { + logAndFail("failed to create PyKMIP deployment: %v", err) + } + + err = waitForDeploymentComplete(f.ClientSet, kmipDeploymentName, cephCSINamespace, deployTimeout) + Expect(err).ShouldNot(HaveOccurred()) + + uid, err := createKMIPKey(f, deployTimeout) + Expect(err).ShouldNot(HaveOccurred()) + + // the kmip KMS provider rejects a credentials Secret with missing or + // unknown keys, these four are exactly what it expects + err = createKMIPSecret(f, kmipCredentialsName, map[string]string{ + "CA_CERT": certs.caCert, + "CLIENT_CERT": certs.clientCert, + "CLIENT_KEY": certs.clientKey, + "UNIQUE_IDENTIFIER": uid, + }) + Expect(err).ShouldNot(HaveOccurred()) +} + +// deleteKMIP removes the PyKMIP server and the Secrets that deployKMIP +// created, also when a failed setup only created some of them. +func deleteKMIP() { + data, err := replaceNamespaceInTemplate(vaultExamplePath + pykmipManifest) + if err != nil { + logAndFail("failed to read content from %s: %v", vaultExamplePath+pykmipManifest, err) + } + err = retryKubectlInput(cephCSINamespace, kubectlDelete, data, deployTimeout) + if err != nil { + logAndFail("failed to delete PyKMIP deployment: %v", err) + } + + err = retryKubectlArgs( + cephCSINamespace, + kubectlDelete, + deployTimeout, + "secret", + kmipCertsSecretName, + kmipCredentialsName, + "--ignore-not-found=true") + Expect(err).ShouldNot(HaveOccurred()) +} + +// createKMIPSecret creates a Secret with the given content, replacing a +// leftover Secret with the same name from an earlier run. +func createKMIPSecret(f *framework.Framework, name string, data map[string]string) error { + err := retryKubectlArgs( + cephCSINamespace, + kubectlDelete, + deployTimeout, + "secret", + name, + "--ignore-not-found=true") + if err != nil { + return fmt.Errorf("failed to delete Secret %q: %w", name, err) + } + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + StringData: data, + } + _, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create Secret %q: %w", name, err) + } + + return nil +} + +// createKMIPKey runs the key creating script inside the PyKMIP pod until it +// succeeds and returns the unique identifier of the created key. Retries +// leave extra keys on the server, which is harmless because the key is +// addressed by the returned unique identifier. +func createKMIPKey(f *framework.Framework, deployTimeout int) (string, error) { + opt := metav1.ListOptions{ + LabelSelector: "app=kmip", + } + uid := "" + timeout := time.Duration(deployTimeout) * time.Minute + err := wait.PollUntilContextTimeout(context.TODO(), poll, timeout, true, func(_ context.Context) (bool, error) { + stdOut, stdErr := execCommandInPodAndAllowFail(f, "python3 /etc/pykmip/create_key.py", cephCSINamespace, &opt) + out := strings.TrimSpace(stdOut) + if out == "" { + framework.Logf("creating the KMIP key has not succeeded yet: %v", stdErr) + + return false, nil + } + + // the unique identifier is on the last line of the output + lines := strings.Split(out, "\n") + uid = strings.TrimSpace(lines[len(lines)-1]) + + return true, nil + }) + if err != nil { + return "", fmt.Errorf("failed to create a key on the KMIP server: %w", err) + } + + return uid, nil +} + +// generateKMIPCertificates returns a fresh CA, a server certificate for the +// DNS names of the kmip Service in the given namespace, and one client +// certificate with a single CN and the clientAuth extended key usage, which +// PyKMIP requires of its clients. +func generateKMIPCertificates(namespace string) (*kmipCertificates, error) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("failed to generate the CA key: %w", err) + } + + caTemplate := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "ceph-csi-kmip-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("failed to create the CA certificate: %w", err) + } + caCert, err := x509.ParseCertificate(caDER) + if err != nil { + return nil, fmt.Errorf("failed to parse the CA certificate: %w", err) + } + + serverCert, serverKey, err := signKMIPLeaf(caCert, caKey, &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: kmipDeploymentName}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{ + kmipDeploymentName, + kmipDeploymentName + "." + namespace, + kmipDeploymentName + "." + namespace + ".svc.cluster.local", + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to create the server certificate: %w", err) + } + + clientCert, clientKey, err := signKMIPLeaf(caCert, caKey, &x509.Certificate{ + SerialNumber: big.NewInt(3), + Subject: pkix.Name{CommonName: "ceph-csi-kmip-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }) + if err != nil { + return nil, fmt.Errorf("failed to create the client certificate: %w", err) + } + + return &kmipCertificates{ + caCert: pemEncodeCertificate(caDER), + serverCert: serverCert, + serverKey: serverKey, + clientCert: clientCert, + clientKey: clientKey, + }, nil +} + +// signKMIPLeaf creates a key pair for the given template, signs the +// certificate with the CA and returns both PEM encoded. +func signKMIPLeaf( + caCert *x509.Certificate, + caKey *ecdsa.PrivateKey, + template *x509.Certificate, +) (string, string, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", "", fmt.Errorf("failed to generate a key: %w", err) + } + + der, err := x509.CreateCertificate(rand.Reader, template, caCert, &key.PublicKey, caKey) + if err != nil { + return "", "", fmt.Errorf("failed to create a certificate: %w", err) + } + + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return "", "", fmt.Errorf("failed to marshal a key: %w", err) + } + + keyPEM := string(pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: keyDER, + })) + + return pemEncodeCertificate(der), keyPEM, nil +} + +func pemEncodeCertificate(der []byte) string { + return string(pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: der, + })) +} diff --git a/e2e/kms.go b/e2e/kms.go index 7538446ae81..00c6e127e28 100644 --- a/e2e/kms.go +++ b/e2e/kms.go @@ -65,6 +65,13 @@ var ( provider: "secrets-metadata", } + // kmipKMS relies on one static key shared by all volumes of the KMS + // configuration, so there is no per-volume passphrase to read back + // or to destroy. + kmipKMS = &simpleKMS{ + provider: "kmip", + } + vaultKMS = &vaultConfig{ simpleKMS: &simpleKMS{ provider: "vault", diff --git a/examples/kms/vault/pykmip.yaml b/examples/kms/vault/pykmip.yaml new file mode 100644 index 00000000000..241cc0c5dbf --- /dev/null +++ b/examples/kms/vault/pykmip.yaml @@ -0,0 +1,129 @@ +# PyKMIP server configuration for minikube +# This is not part of ceph-csi project, used only +# for e2e testing of integration with a KMIP KMS. +# +# The TLS certificates in the ceph-csi-kmip-certs Secret and the +# ceph-csi-kmip-credentials Secret with the created key are generated +# by the e2e tests, see e2e/deploy-kmip.go. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pykmip-config +data: + server.conf: | + [server] + hostname=0.0.0.0 + port=5696 + certificate_path=/etc/pykmip/certs/server.crt + key_path=/etc/pykmip/certs/server.key + ca_path=/etc/pykmip/certs/ca.crt + auth_suite=TLS1.2 + enable_tls_client_auth=True + logging_level=DEBUG + database_path=/tmp/pykmip.sqlite + pykmip.conf: | + [client] + host=127.0.0.1 + port=5696 + certfile=/etc/pykmip/certs/client.crt + keyfile=/etc/pykmip/certs/client.key + ca_certs=/etc/pykmip/certs/ca.crt + cert_reqs=CERT_REQUIRED + ssl_version=PROTOCOL_TLSv1_2 + do_handshake_on_connect=True + suppress_ragged_eofs=True + create_key.py: | + # Creates the AES key served to ceph-csi and prints its unique + # identifier. The client certificate is the same one ceph-csi uses, + # so the created key is owned by the identity that later fetches it. + from kmip.pie.client import ProxyKmipClient + from kmip.core import enums + + client = ProxyKmipClient(config_file='/etc/pykmip/pykmip.conf') + with client: + uid = client.create( + enums.CryptographicAlgorithm.AES, + 256, + name='ceph-csi-fscrypt', + cryptographic_usage_mask=[ + enums.CryptographicUsageMask.ENCRYPT, + enums.CryptographicUsageMask.DECRYPT, + ]) + client.activate(uid) + print(uid) + +--- +apiVersion: v1 +kind: Service +metadata: + name: kmip + labels: + app: kmip +spec: + ports: + - name: kmip + port: 5696 + selector: + app: kmip + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kmip + labels: + app: kmip +spec: + replicas: 1 + selector: + matchLabels: + app: kmip + template: + metadata: + labels: + app: kmip + spec: + containers: + - name: pykmip + image: docker.io/library/python:3.11-slim + imagePullPolicy: "IfNotPresent" + # PyKMIP 0.10.0 is the last release and every dependency is + # pinned to the last version it works with: + # - Python <= 3.11: ssl.wrap_socket is gone in 3.12 + # - SQLAlchemy < 2.0: PyKMIP uses the 1.x query API + # - cryptography 41.0.7: later releases removed the legacy + # ciphers (TripleDES) that PyKMIP imports, and starting the + # server dies with SIGILL + # Bump these pins only together with a PyKMIP release that + # supports them. + command: + - sh + - -c + - >- + pip install --no-cache-dir pykmip==0.10.0 'SQLAlchemy<2.0' + cryptography==41.0.7 && + exec pykmip-server -f /etc/pykmip/server.conf + -l /tmp/pykmip-server.log + ports: + - containerPort: 5696 + name: kmip + readinessProbe: + tcpSocket: + port: 5696 + # the pip install runs before the server starts listening + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 30 + volumeMounts: + - name: pykmip-config + mountPath: /etc/pykmip + - name: pykmip-certs + mountPath: /etc/pykmip/certs + volumes: + - name: pykmip-config + configMap: + name: pykmip-config + - name: pykmip-certs + secret: + secretName: ceph-csi-kmip-certs From 783cf07e59cdfc206f22dc14ec5e67d41ded04ff Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 31 Aug 2026 16:40:43 +0200 Subject: [PATCH 5/8] e2e: test CephFS fscrypt encryption with a KMIP KMS Add a kmip-fscrypt-test KMS configuration pointing at the PyKMIP Service, with USE_CRYPTO_RPC disabled so GetSecret may fetch the key material, and run the encrypted PVC and app binding spec against it. The KMIP server is only deployed for fscrypt test runs, keeping the PyPI dependency out of every other CI job. Assisted-by: Claude Code Signed-off-by: David Mohren --- e2e/cephfs.go | 7 +++++++ examples/kms/vault/kms-config.yaml | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index fd2a25fe8ad..087af1f2d00 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -244,6 +244,9 @@ var _ = Describe(cephfsType, func() { } if !skipVault { deployVault(f.ClientSet, deployTimeout) + if testCephFSFscrypt { + deployKMIP(f, deployTimeout) + } } else { err = createEmptyKMSConfigMap(f.ClientSet, cephCSINamespace) if err != nil { @@ -308,6 +311,9 @@ var _ = Describe(cephfsType, func() { } if !skipVault { deleteVault() + if testCephFSFscrypt { + deleteKMIP() + } } if !cephFSDeleted { @@ -600,6 +606,7 @@ var _ = Describe(cephfsType, func() { "vault-test": vaultKMS, "vault-tokens-test": vaultTokensKMS, "vault-tenant-sa-test": vaultTenantSAKMS, + "kmip-fscrypt-test": kmipKMS, } for kmsID, kmsConf := range kmsToTest { diff --git a/examples/kms/vault/kms-config.yaml b/examples/kms/vault/kms-config.yaml index 7529cc99919..b9f959152f3 100644 --- a/examples/kms/vault/kms-config.yaml +++ b/examples/kms/vault/kms-config.yaml @@ -109,6 +109,14 @@ data: "READ_TIMEOUT": 10, "WRITE_TIMEOUT": 10 }, + "kmip-fscrypt-test": { + "KMS_PROVIDER": "kmip", + "KMIP_ENDPOINT": "kmip.default.svc.cluster.local:5696", + "TLS_SERVER_NAME": "kmip.default.svc.cluster.local", + "USE_CRYPTO_RPC": "false", + "READ_TIMEOUT": 10, + "WRITE_TIMEOUT": 10 + }, "azure-test": { "KMS_PROVIDER": "azure-kv", "AZURE_CERT_SECRET_NAME": "ceph-csi-azure-credentials", From 6f0a8d66c4c77b06b1ac650b45fd06c6a456ef77 Mon Sep 17 00:00:00 2001 From: David Mohren Date: Wed, 9 Sep 2026 11:47:59 +0200 Subject: [PATCH 6/8] e2e: cover fscrypt clones and snapshot-backed volumes with KMIP Run the encrypted PVC-PVC clone and the encrypted snapshot-backed volume specs against kmip-fscrypt-test. Both mount a volume that inherits the fscrypt metadata of its parent, a case the plain encrypted PVC spec does not cover. Requested: https://github.com/ceph/ceph-csi/pull/6521#issuecomment-5583279749 Assisted-by: Claude Code Signed-off-by: David Mohren --- e2e/cephfs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 087af1f2d00..327d856eecd 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -2000,7 +2000,7 @@ var _ = Describe(cephfsType, func() { }) if testCephFSFscrypt { - for _, kmsID := range []string{"secrets-metadata-test", "vault-test"} { + for _, kmsID := range []string{"secrets-metadata-test", "vault-test", "kmip-fscrypt-test"} { It("checking encrypted snapshot-backed volume with KMS "+kmsID, func() { err := deleteResource(cephFSExamplePath + "storageclass.yaml") if err != nil { @@ -2609,6 +2609,7 @@ var _ = Describe(cephfsType, func() { kmsToTest := map[string]kmsConfig{ "secrets-metadata-test": secretsMetadataKMS, "vault-test": vaultKMS, + "kmip-fscrypt-test": kmipKMS, } for kmsID, kmsConf := range kmsToTest { It("create an encrypted PVC-PVC clone and bind it to an app with "+kmsID, func() { From 31526b2e7cb74244837945149ac316ec98e7911a Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 14 Sep 2026 17:35:08 +0200 Subject: [PATCH 7/8] e2e: DO NOT MERGE, enable the fscrypt tests by default Signed-off-by: David Mohren --- e2e/e2e_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 5e297a65e6c..8811525f40e 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -38,9 +38,9 @@ func init() { flag.BoolVar(&deployNFS, "deploy-nfs", false, "deploy nfs csi driver") flag.BoolVar(&deployNVMeoF, "deploy-nvmeof", false, "deploy nvmeof csi driver") flag.BoolVar(&testCephFS, "test-cephfs", true, "test cephFS csi driver") - flag.BoolVar(&testCephFSFscrypt, "test-cephfs-fscrypt", false, "test CephFS csi driver fscrypt support") + flag.BoolVar(&testCephFSFscrypt, "test-cephfs-fscrypt", true, "test CephFS csi driver fscrypt support") flag.BoolVar(&testRBD, "test-rbd", true, "test rbd csi driver") - flag.BoolVar(&testRBDFSCrypt, "test-rbd-fscrypt", false, "test rbd csi driver fscrypt support") + flag.BoolVar(&testRBDFSCrypt, "test-rbd-fscrypt", true, "test rbd csi driver fscrypt support") flag.BoolVar(&testNBD, "test-nbd", false, "test rbd csi driver with rbd-nbd mounter") flag.BoolVar(&testNFS, "test-nfs", false, "test nfs csi driver") flag.BoolVar(&testNVMeoF, "test-nvmeof", true, "test nvmeof csi driver") From efca374842e951d8a51eb570c6f97a2466d70015 Mon Sep 17 00:00:00 2001 From: David Mohren Date: Mon, 14 Sep 2026 22:24:49 +0200 Subject: [PATCH 8/8] e2e: DO NOT MERGE, align the CephFS user OSD caps with the docs exclusive lock needs `x` caps which were not given to the e2e CephX user Signed-off-by: David Mohren --- e2e/ceph_user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/ceph_user.go b/e2e/ceph_user.go index e6484b9af98..73aea496f5a 100644 --- a/e2e/ceph_user.go +++ b/e2e/ceph_user.go @@ -81,7 +81,7 @@ func cephFSNodePluginCaps() []string { caps := []string{ "mon", "'allow r'", "mgr", "'allow rw'", - "osd", "'allow rw tag cephfs *=*'", + "osd", "'allow rwx tag cephfs metadata=*, allow rw tag cephfs data=*'", "mds", "'allow rw'", }