From 5d3bab09dd03b9be0937000e6c882b4041115ef6 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Thu, 6 Aug 2026 08:48:35 +0200 Subject: [PATCH 1/2] fix(RHDHBUGS-3569): scope Deployment/StatefulSet cache to operator-managed objects The metadata informers for Deployments and StatefulSets cached PartialObjectMetadata for every object of those kinds cluster-wide, causing OOM on large multi-tenant clusters (~800 MB heap with 2000 users). Add unconditional cache.ByObject label selectors for Deployments and StatefulSets filtered on app.kubernetes.io/name=backstage, so only operator-managed objects are cached. Replace WatchesMetadata with Owns() for idiomatic ownerReference-based event mapping. Assisted-by: Claude --- cmd/main.go | 42 ++++++++++++++++----- internal/controller/watchers.go | 65 +++------------------------------ 2 files changed, 38 insertions(+), 69 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 739af3915..c852eb84a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,6 +10,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -27,6 +28,7 @@ import ( bsv1 "github.com/redhat-developer/rhdh-operator/api/v1alpha5" "github.com/redhat-developer/rhdh-operator/internal/controller" + "github.com/redhat-developer/rhdh-operator/pkg/utils" openshift "github.com/openshift/api/route/v1" //+kubebuilder:scaffold:imports @@ -137,10 +139,34 @@ func main() { // With many resources, managedFields can consume 70%+ of the informer cache heap. mgrOpts.Cache.DefaultTransform = cache.TransformStripManagedFields() - // Configure cache to only watch labeled Secrets and ConfigMaps if flag is enabled + // Restrict Deployment/StatefulSet cache to operator-managed objects only. + // The operator only needs Deployments/StatefulSets it creates, and those always + // carry app.kubernetes.io/name=backstage. Without this filter, the metadata + // informers cache every Deployment and StatefulSet cluster-wide, which on + // large multi-tenant clusters consumes hundreds of MB of heap. + backstageLabelSelector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: map[string]string{ + utils.BackstageAppLabel: utils.BackstageAppName, + }, + }) + if err != nil { + setupLog.Error(err, "failed to create backstage label selector") + os.Exit(1) + } + + mgrOpts.Cache.ByObject = map[client.Object]cache.ByObject{ + &appsv1.Deployment{}: { + Label: backstageLabelSelector, + }, + &appsv1.StatefulSet{}: { + Label: backstageLabelSelector, + }, + } + + // Optionally restrict Secret/ConfigMap cache to labeled objects. if enableCacheLabelFilter { setupLog.Info("Enabling cache label filter for Secrets and ConfigMaps") - labelSelector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + extConfigLabelSelector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{ "rhdh.redhat.com/external-config": "true", }, @@ -150,13 +176,11 @@ func main() { os.Exit(1) } - mgrOpts.Cache.ByObject = map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Label: labelSelector, - }, - &corev1.ConfigMap{}: { - Label: labelSelector, - }, + mgrOpts.Cache.ByObject[&corev1.Secret{}] = cache.ByObject{ + Label: extConfigLabelSelector, + } + mgrOpts.Cache.ByObject[&corev1.ConfigMap{}] = cache.ByObject{ + Label: extConfigLabelSelector, } } diff --git a/internal/controller/watchers.go b/internal/controller/watchers.go index 13a7b9cab..035d2fa6e 100644 --- a/internal/controller/watchers.go +++ b/internal/controller/watchers.go @@ -7,6 +7,7 @@ import ( "github.com/redhat-developer/rhdh-operator/api" "github.com/redhat-developer/rhdh-operator/pkg/model" "github.com/redhat-developer/rhdh-operator/pkg/utils" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -74,53 +75,10 @@ func (r *BackstageReconciler) addWatchers(b *builder.Builder) error { })) } - // Watch Deployment for the Backstage CR status if enabled - labelPred, err := predicate.LabelSelectorPredicate(metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: utils.BackstageAppLabel, - Values: []string{utils.BackstageAppName}, - Operator: metav1.LabelSelectorOpIn, - }, - }, - }) - if err != nil { - return fmt.Errorf("failed to construct the predicate for backstage deployment. This should not happen: %w", err) - } - - commonPreds := builder.WithPredicates(labelPred, predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { return true }, - UpdateFunc: func(e event.UpdateEvent) bool { return true }, - CreateFunc: func(e event.CreateEvent) bool { return true }, - }) - - metaFor := func(kind string) *metav1.PartialObjectMetadata { - m := &metav1.PartialObjectMetadata{} - m.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apps", - Version: "v1", - Kind: kind, - }) - return m - } - - // Deployment - b.WatchesMetadata( - metaFor("Deployment"), - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { - return r.requestByAppLabels(ctx, o) - }), - commonPreds, - ) - - // StatefulSet - b.WatchesMetadata( - metaFor("StatefulSet"), - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { - return r.requestByAppLabels(ctx, o) - }), - commonPreds, - ) + // Watch operator-owned Deployments and StatefulSets for status tracking. + // Owns() maps events back to the owning Backstage CR via ownerReferences. + b.Owns(&appsv1.Deployment{}). + Owns(&appsv1.StatefulSet{}) return nil } @@ -183,16 +141,3 @@ func (r *BackstageReconciler) requestByExtConfigLabel(ctx context.Context, objec return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: backstage.Name, Namespace: object.GetNamespace()}}} } - -// requestByAppLabels returns a request with current Namespace and Backstage Object name taken from label -func (r *BackstageReconciler) requestByAppLabels(ctx context.Context, object client.Object) []reconcile.Request { - lg := log.FromContext(ctx) - - backstageName := object.GetLabels()[utils.BackstageInstanceLabel] - if object.GetLabels()[utils.BackstageAppLabel] == "" || backstageName == "" { - return []reconcile.Request{} - } - - lg.V(1).Info("enqueuing reconcile on change of ", "kind: ", object.GetObjectKind().GroupVersionKind().Kind, "name: ", object.GetName(), "namespace: ", object.GetNamespace()) - return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: backstageName, Namespace: object.GetNamespace()}}} -} From c08f08926d0eef149355201a92393b04aa4bb3e1 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Thu, 6 Aug 2026 09:14:50 +0200 Subject: [PATCH 2/2] fix(RHDHBUGS-3569): preserve debug logging for Deployment/StatefulSet watch events Add predicate-based V(1) logging to the Owns() watchers for Deployments and StatefulSets, preserving the "enqueuing reconcile on change of" debug messages that were previously emitted by requestByAppLabels. Each log entry now also includes the event type (create, update, delete). Assisted-by: Claude --- internal/controller/watchers.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/controller/watchers.go b/internal/controller/watchers.go index 035d2fa6e..e3b153565 100644 --- a/internal/controller/watchers.go +++ b/internal/controller/watchers.go @@ -77,8 +77,30 @@ func (r *BackstageReconciler) addWatchers(b *builder.Builder) error { // Watch operator-owned Deployments and StatefulSets for status tracking. // Owns() maps events back to the owning Backstage CR via ownerReferences. - b.Owns(&appsv1.Deployment{}). - Owns(&appsv1.StatefulSet{}) + logEvent := func(eventType string, obj client.Object) { + log.Log.V(1).Info("enqueuing reconcile on change of", + "event", eventType, + "kind", obj.GetObjectKind().GroupVersionKind().Kind, + "name", obj.GetName(), + "namespace", obj.GetNamespace()) + } + commonPreds := builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + logEvent("create", e.Object) + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + logEvent("update", e.ObjectNew) + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { + logEvent("delete", e.Object) + return true + }, + }) + + b.Owns(&appsv1.Deployment{}, commonPreds). + Owns(&appsv1.StatefulSet{}, commonPreds) return nil }