ToSquash: Use cache to avoid overload Prometheus

Author: hongkailiu

pkg/clusterconditions/promql/alerts.go

@@ -3,11 +3,15 @@ package promql
 import (
 	"context"
 	"fmt"
+	"sync"
+	"time"
 
 	"github.com/prometheus/client_golang/api"
 	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/config"
 
+	"k8s.io/klog/v2"
+
 	"github.com/openshift/cluster-version-operator/pkg/clusterconditions"
 )
 
@@ -22,32 +26,49 @@ func NewAlertGetter(promQLTarget clusterconditions.PromQLTarget) Getter {
 	if !ok {
 		panic("invalid condition type")
 	}
-	return &ocAlertGetter{promQL: v}
+	return &ocAlertGetter{promQL: v, expiration: 1 * time.Minute}
 }
 
 type ocAlertGetter struct {
 	promQL *PromQL
+
+	mutex       sync.Mutex
+	cached      prometheusv1.AlertsResult
+	expiration  time.Duration
+	lastRefresh time.Time
 }
 
 func (o *ocAlertGetter) Get(ctx context.Context) (prometheusv1.AlertsResult, error) {
-	var ret prometheusv1.AlertsResult
+	if time.Now().After(o.lastRefresh.Add(o.expiration)) {
+		if err := o.refresh(ctx); err != nil {
+			klog.Errorf("Failed to refresh alerts, using stale cache instead: %v", err)
+		}
+	}
+	return o.cached, nil
+}
+
+func (o *ocAlertGetter) refresh(ctx context.Context) error {
+	o.mutex.Lock()
+	defer o.mutex.Unlock()
+
+	klog.Errorf("refresh alerts ...")
 	p := o.promQL
 	host, err := p.Host(ctx)
 	if err != nil {
-		return ret, fmt.Errorf("failure determine thanos IP: %w", err)
+		return fmt.Errorf("failure determine thanos IP: %w", err)
 	}
 	p.url.Host = host
 	clientConfig := api.Config{Address: p.url.String()}
 
 	if roundTripper, err := config.NewRoundTripperFromConfig(p.HTTPClientConfig, "cluster-conditions"); err == nil {
 		clientConfig.RoundTripper = roundTripper
 	} else {
-		return ret, fmt.Errorf("creating PromQL round-tripper: %w", err)
+		return fmt.Errorf("creating PromQL round-tripper: %w", err)
 	}
 
 	promqlClient, err := api.NewClient(clientConfig)
 	if err != nil {
-		return ret, fmt.Errorf("creating PromQL client: %w", err)
+		return fmt.Errorf("creating PromQL client: %w", err)
 	}
 
 	client := &statusCodeNotImplementedForPostClient{
@@ -63,5 +84,12 @@ func (o *ocAlertGetter) Get(ctx context.Context) (prometheusv1.AlertsResult, err
 		defer cancel()
 	}
 
-	return v1api.Alerts(queryContext)
+	r, err := v1api.Alerts(queryContext)
+	if err != nil {
+		return fmt.Errorf("failed to get alerts: %w", err)
+	}
+	o.cached = r
+	o.lastRefresh = time.Now()
+	klog.Errorf("refreshed alerts")
+	return nil
 }