Add an opt-in pod-template-generation label for TopologySpreadConstraints.matchLabelKeys

api/v1beta2/foundationdb_labels.go

@@ -25,6 +25,27 @@ const (
 	// pod spec.
 	LastSpecKey = "foundationdb.org/last-applied-spec"
 
+	// PodTemplateGenerationLabel is the label name carrying a content hash
+	// that identifies a Pod's "generation" — the cohort of pods of the same
+	// process class that render to the same desired PodSpec. Its value is the
+	// first 16 hex characters of a SHA-256 over the canonical rendered
+	// PodSpec for the class, computed from the same source LastSpecKey uses
+	// to drive pod replacement, so the label rotates exactly when the
+	// rendered PodSpec for the class changes.
+	//
+	// Suitable for use with TopologySpreadConstraints.matchLabelKeys to scope
+	// spread to the cohort of pods sharing the same generation. Emission is
+	// opt-in via LabelConfig.IncludePodTemplateGenerationLabel.
+	PodTemplateGenerationLabel = "foundationdb.org/pod-template-generation"
+
+	// PodTemplateGenerationRemovalValue is the PodTemplateGenerationLabel value
+	// stamped on pods whose process group is marked for removal. It buckets
+	// these doomed pods away from any live generation so they do not participate
+	// in — and therefore cannot skew — the surviving cohort's topology spread.
+	// It is not a 16-hex SHA-256 prefix, so it can never collide with a real
+	// generation value.
+	PodTemplateGenerationRemovalValue = "marked-for-removal"
+
 	// LastConfigMapKey provides the annotation name we use to store the hash of the
 	// config map.
 	LastConfigMapKey = "foundationdb.org/last-applied-config-map"

api/v1beta2/foundationdbcluster_types.go

@@ -2565,6 +2565,15 @@ type LabelConfig struct {
 	// by the match labels.
 	// Deprecated: This setting will be removed in the next major release.
 	FilterOnOwnerReferences *bool `json:"filterOnOwnerReference,omitempty"`
+
+	// IncludePodTemplateGenerationLabel determines whether the operator stamps
+	// each Pod at creation time with the "foundationdb.org/pod-template-generation"
+	// label, a class-scoped hash of the rendered PodSpec. The label is intended
+	// for use with topologySpreadConstraints.matchLabelKeys so that a rolling
+	// update spreads the new pod generation independently of the old one. It is
+	// applied only when a Pod is created, so enabling it on an existing cluster
+	// populates the label gradually as Pods are recreated. Defaults to false.
+	IncludePodTemplateGenerationLabel *bool `json:"includePodTemplateGenerationLabel,omitempty"`
 }
 
 // PublicIPSource models options for how a pod gets its public IP.
@@ -2684,6 +2693,13 @@ func (cluster *FoundationDBCluster) ShouldFilterOnOwnerReferences() bool {
 	return ptr.Deref(cluster.Spec.LabelConfig.FilterOnOwnerReferences, false)
 }
 
+// ShouldIncludePodTemplateGenerationLabel returns whether the cluster wants
+// Pods to be labeled with a content hash of their canonical rendered PodSpec
+// under PodTemplateGenerationLabel. Defaults to false.
+func (cluster *FoundationDBCluster) ShouldIncludePodTemplateGenerationLabel() bool {
+	return ptr.Deref(cluster.Spec.LabelConfig.IncludePodTemplateGenerationLabel, false)
+}
+
 // SkipProcessGroup checks if a ProcessGroupStatus should be skipped during reconciliation.
 func (cluster *FoundationDBCluster) SkipProcessGroup(processGroup *ProcessGroupStatus) bool {
 	if processGroup == nil {

config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml

@@ -420,6 +420,8 @@ spec:
                 properties:
                   filterOnOwnerReference:
                     type: boolean
+                  includePodTemplateGenerationLabel:
+                    type: boolean
                   matchLabels:
                     additionalProperties:
                       type: string

controllers/add_pods.go

@@ -143,6 +143,18 @@ func (a addPods) reconcile(
 			pod.Annotations[fdbv1beta2.PublicIPAnnotation] = ip
 		}
 
+		createArgs := []any{
+			"processGroupID", processGroup.ProcessGroupID,
+			"processClass", processGroup.ProcessClass,
+		}
+		if cluster.ShouldIncludePodTemplateGenerationLabel() {
+			createArgs = append(createArgs,
+				"podTemplateGeneration",
+				pod.ObjectMeta.Labels[fdbv1beta2.PodTemplateGenerationLabel],
+			)
+		}
+		logger.Info("Creating pod", createArgs...)
+
 		err = r.PodLifecycleManager.CreatePod(logr.NewContext(ctx, logger), r, pod)
 		if err != nil {
 			if errors.IsQuotaExceeded(err) {

controllers/add_pods_test.go

@@ -29,6 +29,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
 )
 
 var _ = Describe("add_pods", func() {
@@ -159,6 +160,35 @@ var _ = Describe("add_pods", func() {
 				})
 			})
 		})
+
+		When("the pod-template-generation label is enabled", func() {
+			BeforeEach(func() {
+				cluster.Spec.LabelConfig.IncludePodTemplateGenerationLabel = ptr.To(true)
+			})
+
+			It("should stamp the created pod with the generation hash", func() {
+				hash, hashErr := internal.GetPodGenerationHash(
+					cluster,
+					fdbv1beta2.ProcessClassStorage,
+				)
+				Expect(hashErr).NotTo(HaveOccurred())
+				expectNewPodToHaveGenerationLabel(newPods, newProcessGroupID, hash)
+			})
+
+			When("the process group is being removed", func() {
+				BeforeEach(func() {
+					processGroupWithoutPod.MarkForRemoval()
+				})
+
+				It("should stamp the created pod with the removal sentinel", func() {
+					expectNewPodToHaveGenerationLabel(
+						newPods,
+						newProcessGroupID,
+						fdbv1beta2.PodTemplateGenerationRemovalValue,
+					)
+				})
+			})
+		})
 	})
 })
 
@@ -191,3 +221,26 @@ func expectNewPodToHaveBeenCreated(
 
 	Expect(podHaveBeenChecked).To(BeTrue())
 }
+
+func expectNewPodToHaveGenerationLabel(
+	newPods *corev1.PodList,
+	newProcessGroup fdbv1beta2.ProcessGroupID,
+	expectedValue string,
+) {
+	expectedPodName := string("operator-test-1-" + newProcessGroup)
+	var podHaveBeenChecked bool
+	for _, pod := range newPods.Items {
+		if pod.Name != expectedPodName {
+			continue
+		}
+
+		Expect(pod.Labels).To(HaveKeyWithValue(
+			fdbv1beta2.PodTemplateGenerationLabel,
+			expectedValue,
+		))
+		podHaveBeenChecked = true
+		break
+	}
+
+	Expect(podHaveBeenChecked).To(BeTrue())
+}

controllers/update_pods.go

@@ -393,17 +393,31 @@ func getPodsToUpdate(
 			continue
 		}
 
-		logger.Info(
-			"Update Pod",
+		updateArgs := []any{
 			"processGroupID",
 			processGroup.ProcessGroupID,
+			"processClass",
+			processGroup.ProcessClass,
+			"podName",
+			pod.Name,
+			"nodeName",
+			pod.Spec.NodeName,
+		}
+		if cluster.ShouldIncludePodTemplateGenerationLabel() {
+			updateArgs = append(updateArgs,
+				"oldPodTemplateGeneration",
+				pod.ObjectMeta.Labels[fdbv1beta2.PodTemplateGenerationLabel],
+			)
+		}
+		updateArgs = append(updateArgs,
 			"reason",
 			fmt.Sprintf(
 				"specHash has changed from %s to %s",
 				specHash,
 				pod.ObjectMeta.Annotations[fdbv1beta2.LastSpecKey],
 			),
 		)
+		logger.Info("Update Pod", updateArgs...)
 
 		podClient, message := reconciler.getPodClient(cluster, pod)
 		if podClient == nil {

docs/cluster_spec.md

@@ -320,6 +320,7 @@ LabelConfig allows customizing labels used by the operator.
 | processGroupIDLabels | ProcessGroupIDLabels provides the labels that we use for the process group ID field. The first label will be used by the operator when filtering resources. | []string | false |
 | processClassLabels | ProcessClassLabels provides the labels that we use for the process class field. The first label will be used by the operator when filtering resources. | []string | false |
 | filterOnOwnerReference | FilterOnOwnerReferences determines whether we should check that resources are owned by the cluster object, in addition to the constraints provided by the match labels. **Deprecated: This setting will be removed in the next major release.** | *bool | false |
+| includePodTemplateGenerationLabel | IncludePodTemplateGenerationLabel determines whether the operator stamps each Pod at creation time with the \"foundationdb.org/pod-template-generation\" label, a class-scoped hash of the rendered PodSpec. The label is intended for use with topologySpreadConstraints.matchLabelKeys so that a rolling update spreads the new pod generation independently of the old one. It is applied only when a Pod is created, so enabling it on an existing cluster populates the label gradually as Pods are recreated. Defaults to false. | *bool | false |
 
 [Back to TOC](#table-of-contents)
 

docs/manual/fault_domains.md

@@ -406,6 +406,93 @@ In particular (to quote the Kubernetes documentation):
 > - only `.spec.minAvailable` can be used, not `.spec.maxUnavailable`.
 > - only an integer value can be used with `.spec.minAvailable`, not a percentage.
 
+## Spreading Pods Across Pod Template Generations
+
+`topologySpreadConstraints` keep pods of a process class spread across fault
+domains. During a rolling update, though, a plain `labelSelector` counts the
+old pods and the new pods as the same group: while the operator recreates pods
+one fault domain at a time, the new pods and the surviving old pods are weighed
+together, which can let the scheduler place several new pods into the same fault
+domain. Because topology spread constraints are only evaluated when a pod is
+scheduled — the scheduler never moves a pod that is already running — that
+imbalance in the new generation is not corrected once the update finishes; it
+persists until those pods happen to be recreated again.
+
+To avoid this you can scope a spread constraint to a single *pod template
+generation* — the cohort of pods of the same process class that render to the
+same `PodSpec`. When you enable it, the operator stamps every pod at creation
+time with the `foundationdb.org/pod-template-generation` label, whose value is a
+hash of the rendered `PodSpec` for that process class. The hash rotates exactly
+when the rendered `PodSpec` for the class changes, so a spec change that triggers
+a rolling update also starts a new generation.
+
+Enable the label in the cluster spec:
+
+```yaml
+apiVersion: apps.foundationdb.org/v1beta2
+kind: FoundationDBCluster
+metadata:
+  name: sample-cluster
+spec:
+  labels:
+    includePodTemplateGenerationLabel: true
+```
+
+Then reference the label from a spread constraint with `matchLabelKeys`. The
+scheduler reads the label's value from the pod being scheduled and only counts
+existing pods that share the same value, so each generation is spread
+independently:
+
+```yaml
+        podTemplate:
+          spec:
+            topologySpreadConstraints:
+              - maxSkew: 1
+                topologyKey: topology.kubernetes.io/zone
+                whenUnsatisfiable: DoNotSchedule
+                # Only consider pods of this cluster and process class ...
+                labelSelector:
+                  matchLabels:
+                    foundationdb.org/fdb-cluster-name: sample-cluster
+                    foundationdb.org/fdb-process-class: storage
+                # ... that also belong to the same pod template generation.
+                matchLabelKeys:
+                  - foundationdb.org/pod-template-generation
+```
+
+`matchLabelKeys` for topology spread requires the
+`MatchLabelKeysInPodTopologySpread` feature to be enabled in your cluster. Do
+not add the generation label to `labelSelector.matchLabels` yourself — the
+scheduler derives its value from the incoming pod automatically.
+
+### Rollout behavior
+
+The label is applied only when a pod is **created**; the operator never patches
+it onto a running pod. This is deliberate: rewriting the value in place on a
+surviving pod would make the scheduler treat the old and new generations as one
+during the update, which is exactly what this label prevents.
+
+As a consequence, enabling `includePodTemplateGenerationLabel` on an existing
+cluster does not label the current pods immediately. The label is populated
+gradually, as pods are recreated for unrelated reasons (upgrades, replacements,
+or spec changes). Until every pod is recreated there will be a mix of labeled
+and unlabeled pods; Kubernetes ignores a `matchLabelKeys` entry for pods that do
+not carry the key, so the constraint degrades gracefully to the broader
+`labelSelector` spread rather than failing to schedule. To have every pod carry
+the label from the start, enable the setting when the cluster is first created.
+
+### Pods marked for removal
+
+When a process group is marked for removal (for example during a replacement),
+any pod the operator creates for it is stamped with the fixed value
+`marked-for-removal` instead of a generation hash. This buckets the doomed pod
+away from every live generation, so it is never counted toward — and therefore
+cannot skew — the spread of the pods that will survive. The sentinel is applied
+only to pods created while the group is already being removed; pods that were
+created earlier keep whatever value they had (the operator never rewrites the
+label in place, per the rule above). Because `marked-for-removal` is not a
+16-character hash, it can never collide with a real generation value.
+
 ## Coordinators
 
 Per default the FDB operator will try to select the best fitting processes to be coordinators.