diff --git a/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-commands.sh b/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-commands.sh index a996c769b4cfb..861622829a72a 100755 --- a/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-commands.sh +++ b/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-commands.sh @@ -160,6 +160,9 @@ dyn_start_time=${start_time} CLUSTER_PREVIOUS_STATE="claim" record_cluster "timers" "status" "claim" loop_count=0 +OCM_RECONCILE_TIMEOUT=${OCM_RECONCILE_TIMEOUT:-900} +install_complete_detected=false +install_complete_time=0 while true; do rosa describe cluster -c "${CLUSTER_ID}" -o json > ${cluster_info_json} CLUSTER_STATE=$(cat ${cluster_info_json} | jq -r '.state') @@ -181,7 +184,39 @@ while true; do else # Stall detection: fail early if state is stuck for too long stall_elapsed=$(( current_time - dyn_start_time )) - if (( stall_elapsed >= STALL_TIMEOUT )) && [[ "${CLUSTER_STATE}" == "installing" || "${CLUSTER_STATE}" == "pending" ]]; then + + # OCM state reconciliation detection: check every iteration (not just every 5th) + # Only when: Classic cluster, state=installing, infra_id is set, past provisioner launch timeout + if [[ "${HOSTED_CP}" != "true" ]] && [[ "${CLUSTER_STATE}" == "installing" ]] && [[ "${install_complete_detected}" != "true" ]]; then + infra_id_check=$(jq -r '.infra_id' "${cluster_info_json}") + installing_elapsed=$(( current_time - dyn_start_time )) + if [[ "${infra_id_check}" != "null" ]] && (( installing_elapsed >= PROVISIONER_LAUNCH_TIMEOUT )); then + install_log_check=$(timeout 60 rosa logs install -c "${CLUSTER_ID}" 2>&1 || true) + if echo "${install_log_check}" | grep -Eiq 'install complete!|install completed successfully'; then + install_complete_detected=true + install_complete_time=${current_time} + log "Install completion detected in logs. Starting OCM reconciliation grace period ($(( OCM_RECONCILE_TIMEOUT / 60 )) minutes)." + log " OCM has ${OCM_RECONCILE_TIMEOUT} seconds to transition cluster state from 'installing' to 'ready'." + fi + fi + fi + + # OCM reconciliation grace period: if install completed, give OCM time to reconcile + if [[ "${install_complete_detected}" == "true" ]]; then + ocm_reconcile_elapsed=$(( current_time - install_complete_time )) + if (( ocm_reconcile_elapsed >= OCM_RECONCILE_TIMEOUT )); then + log "FATAL: OCM state reconciliation failure detected." + log " OpenShift install completed $(( ocm_reconcile_elapsed / 60 )) minutes ago, but OCM state is still '${CLUSTER_STATE}'." + log " OCM reconciliation grace period ($(( OCM_RECONCILE_TIMEOUT / 60 )) minutes) exceeded." + record_cluster "timers" "status" "ocm_state_stall" + FAILED_INSTALL="yes" + break + else + log " OCM reconciliation in progress: $(( ocm_reconcile_elapsed / 60 ))m $(( ocm_reconcile_elapsed % 60 ))s / $(( OCM_RECONCILE_TIMEOUT / 60 ))m" + fi + fi + + if [[ "${install_complete_detected}" != "true" ]] && (( stall_elapsed >= STALL_TIMEOUT )) && [[ "${CLUSTER_STATE}" == "installing" || "${CLUSTER_STATE}" == "pending" ]]; then log "ERROR: Cluster state '${CLUSTER_STATE}' has not changed for $(( stall_elapsed / 60 )) minutes (stall timeout: $(( STALL_TIMEOUT / 60 )) minutes)" log "Cluster appears to be stalled. Failing early." record_cluster "timers" "status" "${CLUSTER_STATE}" @@ -221,20 +256,6 @@ while true; do break fi fi - - # OCM state reconciliation detection: installer completed but OCM state stuck - if [[ "${infra_id_check}" != "null" ]] && (( installing_elapsed >= PROVISIONER_LAUNCH_TIMEOUT )); then - install_log_check=$(timeout 60 rosa logs install -c "${CLUSTER_ID}" 2>&1 || true) - if echo "${install_log_check}" | grep -Eiq 'install complete!|install completed successfully'; then - log "FATAL: OCM state reconciliation failure detected." - log " infra_id is set (provisioner launched) but cluster is still in 'installing' state" - log " after $(( installing_elapsed / 60 )) minutes, despite install logs showing completion." - log " This indicates OCM failed to reconcile the cluster state from 'installing' to 'ready'." - record_cluster "timers" "status" "ocm_state_stall" - FAILED_INSTALL="yes" - break - fi - fi fi fi diff --git a/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-ref.yaml b/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-ref.yaml index 0e9b5cfe6362c..a03695a093c31 100644 --- a/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-ref.yaml +++ b/ci-operator/step-registry/rosa/cluster/wait-ready/cluster/rosa-cluster-wait-ready-cluster-ref.yaml @@ -36,6 +36,9 @@ ref: - name: PROVISIONER_LAUNCH_TIMEOUT default: "900" documentation: Maximum seconds to wait for the Hive provisioner to launch (infra_id to become non-null) after entering the 'installing' state. Only applies to Classic (non-HCP) clusters. Default is 900 (15 minutes). + - name: OCM_RECONCILE_TIMEOUT + default: "900" + documentation: Maximum seconds to wait for OCM to reconcile the cluster state from 'installing' to 'ready' after the OpenShift installer reports completion. This grace period activates only when install completion is detected in logs and prevents the general STALL_TIMEOUT from firing prematurely on long installs. Only applies to Classic (non-HCP) clusters. Default is 900 (15 minutes). documentation: |- Wait the rosa cluster to be ready.