From 60de67f3d35d0ff957196398c3b1627ca0261aa2 Mon Sep 17 00:00:00 2001 From: nhamza Date: Wed, 19 Aug 2026 10:05:50 +0300 Subject: [PATCH] OCPBUGS-105240: TNF: wait for etcd-bootstrap member removal before setup The TNF setup gate in waitForEtcdBootstrapCompleted only checked EtcdRunningInCluster, which the bootstrap teardown controller sets before removing the etcd-bootstrap member (it is the signal bootkube watches to proceed with teardown). TNF setup could therefore start pacemaker while etcd-bootstrap was still a cluster member. The podman-etcd resource agent requires exactly 2 members and refuses to set the learner_node attribute when it sees 3, deadlocking the installation. Gate TNF setup on EtcdBootstrapMemberRemoved in addition, via a new ceohelpers.IsEtcdBootstrapMemberRemoved helper. If the member is not yet removed, return an error and let the job controller startup backoff retry. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ceohelpers/external_etcd_status.go | 26 +++++++++++++++++++ pkg/tnf/operator/job_controllers.go | 16 ++++++++++++ 2 files changed, 42 insertions(+) diff --git a/pkg/operator/ceohelpers/external_etcd_status.go b/pkg/operator/ceohelpers/external_etcd_status.go index 52f9b4263e..8f3307b793 100644 --- a/pkg/operator/ceohelpers/external_etcd_status.go +++ b/pkg/operator/ceohelpers/external_etcd_status.go @@ -15,6 +15,7 @@ import ( const ( OperatorConditionEtcdRunningInCluster = "EtcdRunningInCluster" + OperatorConditionEtcdBootstrapMemberRemoved = "EtcdBootstrapMemberRemoved" OperatorConditionExternalEtcdReadyForTransition = "ExternalEtcdReadyForTransition" OperatorConditionExternalEtcdHasCompletedTransition = "ExternalEtcdHasCompletedTransition" ) @@ -88,6 +89,31 @@ func IsEtcdRunningInCluster(ctx context.Context, operatorClient v1helpers.Static return etcdRunningInCluster, nil } +// IsEtcdBootstrapMemberRemoved checks if the etcd-bootstrap member has been +// removed from the etcd cluster by examining the operator status for the +// EtcdBootstrapMemberRemoved condition. Unlike EtcdRunningInCluster, which is +// set before the bootstrap member is removed, this condition is only set once +// the member removal has been confirmed. +func IsEtcdBootstrapMemberRemoved(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) { + _, opStatus, _, err := operatorClient.GetStaticPodOperatorState() + if err != nil { + klog.Errorf("failed to get static pod operator state: %v", err) + return false, err + } + + if opStatus == nil { + klog.V(2).Info("static pod operator status not yet populated; bootstrap member removal unknown") + return false, nil + } + + bootstrapMemberRemoved := v1helpers.IsOperatorConditionTrue(opStatus.Conditions, OperatorConditionEtcdBootstrapMemberRemoved) + if bootstrapMemberRemoved { + klog.V(4).Infof("etcd-bootstrap member has been removed") + } + + return bootstrapMemberRemoved, nil +} + // HasExternalEtcdCompletedTransition checks if the transition to external etcd process is completed // by examining the operator status for the HasExternalEtcdCompletedTransition condition. func HasExternalEtcdCompletedTransition(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) { diff --git a/pkg/tnf/operator/job_controllers.go b/pkg/tnf/operator/job_controllers.go index 44f7615e63..2dd6d454d7 100644 --- a/pkg/tnf/operator/job_controllers.go +++ b/pkg/tnf/operator/job_controllers.go @@ -469,5 +469,21 @@ func waitForEtcdBootstrapCompleted(ctx context.Context, operatorClient v1helpers return fmt.Errorf("failed to wait for bootstrap to complete: %w", err) } } + + // EtcdRunningInCluster is set before the bootstrap member is removed, as + // it signals bootkube that it can proceed with bootstrap teardown. TNF + // setup must not start pacemaker while etcd-bootstrap is still a member: + // podman-etcd requires exactly 2 members and deadlocks on 3 + // (OCPBUGS-105240). EtcdBootstrapMemberRemoved is only set once the + // member is confirmed gone, so gate on it as well. Returning an error + // here is safe: the caller retries with backoff. + bootstrapMemberRemoved, err := ceohelpers.IsEtcdBootstrapMemberRemoved(ctx, operatorClient) + if err != nil { + return fmt.Errorf("failed to check if etcd-bootstrap member is removed: %w", err) + } + if !bootstrapMemberRemoved { + return fmt.Errorf("etcd-bootstrap member has not been removed yet") + } + return nil }