OCPBUGS-105240: TNF: wait for etcd-bootstrap member removal before setup

pkg/operator/ceohelpers/external_etcd_status.go

@@ -15,6 +15,7 @@ import (
 
 const (
 	OperatorConditionEtcdRunningInCluster               = "EtcdRunningInCluster"
+	OperatorConditionEtcdBootstrapMemberRemoved         = "EtcdBootstrapMemberRemoved"
 	OperatorConditionExternalEtcdReadyForTransition     = "ExternalEtcdReadyForTransition"
 	OperatorConditionExternalEtcdHasCompletedTransition = "ExternalEtcdHasCompletedTransition"
 )
@@ -88,6 +89,31 @@ func IsEtcdRunningInCluster(ctx context.Context, operatorClient v1helpers.Static
 	return etcdRunningInCluster, nil
 }
 
+// IsEtcdBootstrapMemberRemoved checks if the etcd-bootstrap member has been
+// removed from the etcd cluster by examining the operator status for the
+// EtcdBootstrapMemberRemoved condition. Unlike EtcdRunningInCluster, which is
+// set before the bootstrap member is removed, this condition is only set once
+// the member removal has been confirmed.
+func IsEtcdBootstrapMemberRemoved(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) {
+	_, opStatus, _, err := operatorClient.GetStaticPodOperatorState()
+	if err != nil {
+		klog.Errorf("failed to get static pod operator state: %v", err)
+		return false, err
+	}
+
+	if opStatus == nil {
+		klog.V(2).Info("static pod operator status not yet populated; bootstrap member removal unknown")
+		return false, nil
+	}
+
+	bootstrapMemberRemoved := v1helpers.IsOperatorConditionTrue(opStatus.Conditions, OperatorConditionEtcdBootstrapMemberRemoved)
+	if bootstrapMemberRemoved {
+		klog.V(4).Infof("etcd-bootstrap member has been removed")
+	}
+
+	return bootstrapMemberRemoved, nil
+}
+
 // HasExternalEtcdCompletedTransition checks if the transition to external etcd process is completed
 // by examining the operator status for the HasExternalEtcdCompletedTransition condition.
 func HasExternalEtcdCompletedTransition(ctx context.Context, operatorClient v1helpers.StaticPodOperatorClient) (bool, error) {

pkg/tnf/operator/job_controllers.go

@@ -469,5 +469,21 @@ func waitForEtcdBootstrapCompleted(ctx context.Context, operatorClient v1helpers
 			return fmt.Errorf("failed to wait for bootstrap to complete: %w", err)
 		}
 	}
+
+	// EtcdRunningInCluster is set before the bootstrap member is removed, as
+	// it signals bootkube that it can proceed with bootstrap teardown. TNF
+	// setup must not start pacemaker while etcd-bootstrap is still a member:
+	// podman-etcd requires exactly 2 members and deadlocks on 3
+	// (OCPBUGS-105240). EtcdBootstrapMemberRemoved is only set once the
+	// member is confirmed gone, so gate on it as well. Returning an error
+	// here is safe: the caller retries with backoff.
+	bootstrapMemberRemoved, err := ceohelpers.IsEtcdBootstrapMemberRemoved(ctx, operatorClient)
+	if err != nil {
+		return fmt.Errorf("failed to check if etcd-bootstrap member is removed: %w", err)
+	}
+	if !bootstrapMemberRemoved {
+		return fmt.Errorf("etcd-bootstrap member has not been removed yet")
+	}
+
 	return nil
 }