diff --git a/bundle/manifests/gitops-operator.clusterserviceversion.yaml b/bundle/manifests/gitops-operator.clusterserviceversion.yaml index cb42eba81eb..0832ad66c38 100644 --- a/bundle/manifests/gitops-operator.clusterserviceversion.yaml +++ b/bundle/manifests/gitops-operator.clusterserviceversion.yaml @@ -203,7 +203,7 @@ metadata: operatorframework.io/cluster-monitoring: "true" operatorframework.io/suggested-namespace: openshift-gitops-operator operators.openshift.io/infrastructure-features: '["disconnected"]' - operators.operatorframework.io/builder: operator-sdk-v1.35.0 + operators.operatorframework.io/builder: operator-sdk-v1.42.2 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 repository: https://github.com/redhat-developer/gitops-operator support: Red Hat diff --git a/bundle/manifests/openshift-gitops-operator-metrics-monitor-bearer-token_v1_secret.yaml b/bundle/manifests/openshift-gitops-operator-metrics-monitor-bearer-token_v1_secret.yaml deleted file mode 100644 index ab422c66cfb..00000000000 --- a/bundle/manifests/openshift-gitops-operator-metrics-monitor-bearer-token_v1_secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - annotations: - kubernetes.io/service-account.name: openshift-gitops-operator-controller-manager - name: openshift-gitops-operator-metrics-monitor-bearer-token -type: kubernetes.io/service-account-token diff --git a/bundle/manifests/openshift-gitops-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml b/bundle/manifests/openshift-gitops-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml index 3b4e719bc5e..87d125c6904 100644 --- a/bundle/manifests/openshift-gitops-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml +++ b/bundle/manifests/openshift-gitops-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml @@ -6,9 +6,11 @@ metadata: name: openshift-gitops-operator-metrics-monitor spec: endpoints: - - bearerTokenSecret: - key: token - name: openshift-gitops-operator-metrics-monitor-bearer-token + - authorization: + credentials: + key: token + name: openshift-gitops-operator-metrics-monitor-bearer-token + type: Bearer interval: 30s path: /metrics port: metrics diff --git a/cmd/main.go b/cmd/main.go index c3dd5dd4471..0d0c7f137fa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -340,6 +340,14 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Argo CD metrics") os.Exit(1) } + + if err = (&controllers.OperatorMetricsTokenReconciler{ + Client: client, + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Operator metrics token") + os.Exit(1) + } } else { setupLog.Info("Monitoring API not found, skipping Argo CD metrics controller setup") } diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index c601f41a47c..76af779b912 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -1,14 +1,5 @@ --- apiVersion: v1 -kind: Secret -metadata: - name: metrics-monitor-bearer-token - namespace: openshift-gitops-operator - annotations: - kubernetes.io/service-account.name: openshift-gitops-operator-controller-manager -type: kubernetes.io/service-account-token ---- -apiVersion: v1 kind: ConfigMap metadata: annotations: @@ -31,9 +22,11 @@ spec: matchLabels: control-plane: gitops-operator endpoints: - - bearerTokenSecret: - name: openshift-gitops-operator-metrics-monitor-bearer-token - key: token + - authorization: + type: Bearer + credentials: + name: openshift-gitops-operator-metrics-monitor-bearer-token + key: token interval: 30s path: /metrics port: metrics diff --git a/controllers/argocd_controller.go b/controllers/argocd_controller.go index 0f82aae61e7..a07a4e62ce4 100644 --- a/controllers/argocd_controller.go +++ b/controllers/argocd_controller.go @@ -45,16 +45,16 @@ import ( ) const ( - argocdNS = "openshift-gitops" - depracatedArgoCDNS = "openshift-pipelines-app-delivery" - consoleLinkName = "argocd" - argocdRouteName = "openshift-gitops-server" - iconFilePath = "/argo.png" - operatorPodNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + argocdNS = "openshift-gitops" + depracatedArgoCDNS = "openshift-pipelines-app-delivery" + consoleLinkName = "argocd" + argocdRouteName = "openshift-gitops-server" + iconFilePath = "/argo.png" ) var ( - encodedArgoImage string + operatorPodNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + encodedArgoImage string //go:embed argocd/img/argo.png argoImage []byte diff --git a/controllers/argocd_metrics_controller.go b/controllers/argocd_metrics_controller.go index 0abe9a1ced8..80221f178f0 100644 --- a/controllers/argocd_metrics_controller.go +++ b/controllers/argocd_metrics_controller.go @@ -20,7 +20,6 @@ import ( "context" "embed" "fmt" - "os" "path/filepath" "strings" @@ -44,13 +43,11 @@ import ( ) const ( - readRoleNameFormat = "%s-read" - readRoleBindingNameFormat = "%s-prometheus-k8s-read-binding" - alertRuleName = "gitops-operator-argocd-alerts" - dashboardNamespace = "openshift-config-managed" - dashboardFolder = "dashboards" - operatorMetricsServiceName = "openshift-gitops-operator-metrics-service" - operatorMetricsMonitorName = "openshift-gitops-operator-metrics-monitor" + readRoleNameFormat = "%s-read" + readRoleBindingNameFormat = "%s-prometheus-k8s-read-binding" + alertRuleName = "gitops-operator-argocd-alerts" + dashboardNamespace = "openshift-config-managed" + dashboardFolder = "dashboards" ) type ArgoCDMetricsReconciler struct { @@ -182,11 +179,6 @@ func (r *ArgoCDMetricsReconciler) Reconcile(ctx context.Context, request reconci if err != nil { return reconcile.Result{}, err } - - err = r.reconcileOperatorMetricsServiceMonitor(reqLogger) - if err != nil { - return reconcile.Result{}, err - } } else { if exists { namespace.Labels[monitoringLabel] = "false" @@ -369,43 +361,6 @@ func (r *ArgoCDMetricsReconciler) deleteServiceMonitor(name string, namespace st } -func (r *ArgoCDMetricsReconciler) reconcileOperatorMetricsServiceMonitor(reqLogger logr.Logger) error { - - data, err := os.ReadFile(operatorPodNamespacePath) - if err != nil { - if os.IsNotExist(err) { - reqLogger.Info(fmt.Sprintf("Unable to retrieve the operator's running namespace via '%s': you should only see this message when running within unit tests, otherwise it is an error.", operatorPodNamespacePath)) - return nil - } - reqLogger.Error(err, "Error retrieving operator's running namespace") - return err - } - - operatorNS := string(data) - desiredMetricsServerName := operatorMetricsServiceName + "." + operatorNS + ".svc" - - existingServiceMonitor := &monitoringv1.ServiceMonitor{} - err = r.Client.Get(context.TODO(), types.NamespacedName{Name: operatorMetricsMonitorName, Namespace: operatorNS}, existingServiceMonitor) - - if err != nil { - if !errors.IsNotFound(err) { - reqLogger.Error(err, "Error querying for ServiceMonitor", "Namespace", operatorNS, "Name", operatorMetricsMonitorName) - return err - } - - // no svc monitor found, nothing to do - return nil - } - - currentServerName := existingServiceMonitor.Spec.Endpoints[0].TLSConfig.ServerName - if currentServerName == nil || *currentServerName != desiredMetricsServerName { - existingServiceMonitor.Spec.Endpoints[0].TLSConfig.ServerName = &desiredMetricsServerName - return r.Client.Update(context.TODO(), existingServiceMonitor) - } - - return nil -} - func (r *ArgoCDMetricsReconciler) createPrometheusRuleIfAbsent(namespace string, argocd *argoapp.ArgoCD, reqLogger logr.Logger) error { alertRule := newPrometheusRule(namespace) existingAlertRule := &monitoringv1.PrometheusRule{} diff --git a/controllers/operator_metrics_controller.go b/controllers/operator_metrics_controller.go new file mode 100644 index 00000000000..b257c79fc44 --- /dev/null +++ b/controllers/operator_metrics_controller.go @@ -0,0 +1,319 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/go-logr/logr" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + operatorMetricsServiceName = "openshift-gitops-operator-metrics-service" + operatorMetricsMonitorName = "openshift-gitops-operator-metrics-monitor" + operatorMetricsBearerTokenSecretName = "openshift-gitops-operator-metrics-monitor-bearer-token" + operatorControllerSAName = "openshift-gitops-operator-controller-manager" + operatorMetricsTokenExpirySecs = int64(3600) + operatorMetricsTokenExpiry = time.Duration(operatorMetricsTokenExpirySecs) * time.Second + operatorMetricsBearerTokenKey = "token" + operatorMetricsBearerTokenExpiryKey = "expiry" +) + +type serviceAccountTokenRequester interface { + RequestToken(ctx context.Context, namespace, serviceAccountName string, expirationSeconds int64) (token string, expiry time.Time, err error) +} + +type clientServiceAccountTokenRequester struct { + client client.Client +} + +func (r *clientServiceAccountTokenRequester) RequestToken(ctx context.Context, namespace, serviceAccountName string, expirationSeconds int64) (string, time.Time, error) { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceAccountName, + Namespace: namespace, + }, + } + tr := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: ptr.To(expirationSeconds), + }, + } + if err := r.client.SubResource("token").Create(ctx, sa, tr); err != nil { + return "", time.Time{}, err + } + return tr.Status.Token, tr.Status.ExpirationTimestamp.Time, nil +} + +// OperatorMetricsTokenReconciler manages the short-lived bearer token Secret used by the +// operator's ServiceMonitor for Prometheus metrics scraping. +type OperatorMetricsTokenReconciler struct { + Client client.Client + Scheme *runtime.Scheme + TokenRequester serviceAccountTokenRequester +} + +var _ reconcile.Reconciler = &OperatorMetricsTokenReconciler{} + +func (r *OperatorMetricsTokenReconciler) tokenRequester() serviceAccountTokenRequester { + if r.TokenRequester != nil { + return r.TokenRequester + } + return &clientServiceAccountTokenRequester{client: r.Client} +} + +// SetupWithManager sets up the controller with the Manager. +func (r *OperatorMetricsTokenReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("operator-metrics-token"). + For(&monitoringv1.ServiceMonitor{}). + WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetName() == operatorMetricsMonitorName + })). + Complete(r) +} + +//+kubebuilder:rbac:groups="",resources=serviceaccounts/token,resourceNames=openshift-gitops-operator-controller-manager,verbs=create +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;update;patch + +func (r *OperatorMetricsTokenReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + reqLogger := logf.Log.WithName("controller_operator_metrics_token"). + WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + + operatorNS, err := getOperatorNamespace() + if err != nil { + if os.IsNotExist(err) { + reqLogger.Info(fmt.Sprintf("Unable to retrieve the operator's running namespace via '%s': you should only see this message when running within unit tests, otherwise it is an error.", operatorPodNamespacePath)) + return reconcile.Result{}, nil + } + reqLogger.Error(err, "Error retrieving operator's running namespace") + return reconcile.Result{}, err + } + + if request.Namespace != operatorNS || request.Name != operatorMetricsMonitorName { + return reconcile.Result{}, nil + } + + serviceMonitor := &monitoringv1.ServiceMonitor{} + if err := r.Client.Get(ctx, request.NamespacedName, serviceMonitor); err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + reqLogger.Error(err, "Error querying for ServiceMonitor") + return reconcile.Result{}, err + } + + if err := r.reconcileServiceMonitor(ctx, serviceMonitor, operatorNS, reqLogger); err != nil { + return reconcile.Result{}, err + } + + requeueAfter, err := r.reconcileBearerTokenSecret(ctx, operatorNS, reqLogger) + if err != nil { + return reconcile.Result{}, err + } + + if requeueAfter > 0 { + reqLogger.Info("Scheduling bearer token renewal", "after", requeueAfter.String()) + } + return reconcile.Result{RequeueAfter: requeueAfter}, nil +} + +func getOperatorNamespace() (string, error) { + data, err := os.ReadFile(operatorPodNamespacePath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(data)), nil +} + +func (r *OperatorMetricsTokenReconciler) reconcileServiceMonitor(ctx context.Context, serviceMonitor *monitoringv1.ServiceMonitor, operatorNS string, reqLogger logr.Logger) error { + if len(serviceMonitor.Spec.Endpoints) == 0 { + return fmt.Errorf("ServiceMonitor %s has no endpoints", serviceMonitor.Name) + } + + desiredMetricsServerName := operatorMetricsServiceName + "." + operatorNS + ".svc" + endpoint := &serviceMonitor.Spec.Endpoints[0] + + updated := false + if endpoint.BearerTokenSecret != nil { + endpoint.BearerTokenSecret = nil + endpoint.Authorization = &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + }, + } + updated = true + } else if endpoint.Authorization == nil || + endpoint.Authorization.Credentials == nil || + endpoint.Authorization.Credentials.Name != operatorMetricsBearerTokenSecretName || + endpoint.Authorization.Credentials.Key != operatorMetricsBearerTokenKey { + endpoint.Authorization = &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + }, + } + updated = true + } + + if endpoint.TLSConfig != nil && (endpoint.TLSConfig.ServerName == nil || *endpoint.TLSConfig.ServerName != desiredMetricsServerName) { + endpoint.TLSConfig.ServerName = &desiredMetricsServerName + updated = true + } + + if !updated { + return nil + } + + reqLogger.Info("Updating operator metrics ServiceMonitor", + "Namespace", serviceMonitor.Namespace, "Name", serviceMonitor.Name) + return r.Client.Update(ctx, serviceMonitor) +} + +func (r *OperatorMetricsTokenReconciler) reconcileBearerTokenSecret(ctx context.Context, namespace string, reqLogger logr.Logger) (time.Duration, error) { + secret := &corev1.Secret{} + err := r.Client.Get(ctx, types.NamespacedName{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: namespace, + }, secret) + + needsRefresh := false + legacySAToken := false + if errors.IsNotFound(err) { + needsRefresh = true + } else if err != nil { + return 0, err + } else if secret.Type == corev1.SecretTypeServiceAccountToken { + // Keep the legacy Secret until TokenRequest succeeds so scrape auth + // is not interrupted if minting fails. + legacySAToken = true + needsRefresh = true + } else { + expiry, parseErr := parseBearerTokenExpiry(secret.Data[operatorMetricsBearerTokenExpiryKey]) + if parseErr != nil || !time.Now().Before(expiry) { + needsRefresh = true + } else { + requeueAfter := bearerTokenRequeueDuration(expiry) + if requeueAfter <= 0 { + needsRefresh = true + } else { + return requeueAfter, nil + } + } + } + + if !needsRefresh { + return 0, nil + } + + token, expiry, err := r.tokenRequester().RequestToken(ctx, namespace, operatorControllerSAName, operatorMetricsTokenExpirySecs) + if err != nil { + reqLogger.Error(err, "Failed to request service account token") + return 0, err + } + + desiredSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + operatorMetricsBearerTokenKey: []byte(token), + operatorMetricsBearerTokenExpiryKey: []byte(expiry.UTC().Format(time.RFC3339)), + }, + } + + if legacySAToken { + reqLogger.Info("Replacing legacy non-expiring service account token Secret", + "Namespace", namespace, "Name", operatorMetricsBearerTokenSecretName) + if err := r.Client.Delete(ctx, secret); err != nil && !errors.IsNotFound(err) { + return 0, err + } + if err := r.Client.Create(ctx, desiredSecret); err != nil { + return 0, err + } + return bearerTokenRequeueDuration(expiry), nil + } + + existingSecret := &corev1.Secret{} + getErr := r.Client.Get(ctx, types.NamespacedName{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: namespace, + }, existingSecret) + if getErr != nil { + if errors.IsNotFound(getErr) { + reqLogger.Info("Creating metrics monitor bearer token Secret", + "Namespace", namespace, "Name", operatorMetricsBearerTokenSecretName) + if err := r.Client.Create(ctx, desiredSecret); err != nil { + return 0, err + } + return bearerTokenRequeueDuration(expiry), nil + } + return 0, getErr + } + + existingSecret.Type = corev1.SecretTypeOpaque + existingSecret.Data = desiredSecret.Data + reqLogger.Info("Updating metrics monitor bearer token Secret", + "Namespace", namespace, "Name", operatorMetricsBearerTokenSecretName) + if err := r.Client.Update(ctx, existingSecret); err != nil { + return 0, err + } + + return bearerTokenRequeueDuration(expiry), nil +} + +func parseBearerTokenExpiry(expiryData []byte) (time.Time, error) { + return time.Parse(time.RFC3339, string(expiryData)) +} + +// bearerTokenRequeueDuration returns how long to wait before renewing the token. +// Renewal is scheduled after one third of the *actual* remaining lifetime has +// elapsed, so cluster-capped TokenRequest lifetimes still get a renewal. +func bearerTokenRequeueDuration(expiry time.Time) time.Duration { + remaining := time.Until(expiry) + if remaining <= 0 { + return 0 + } + return remaining / 3 +} diff --git a/controllers/operator_metrics_controller_test.go b/controllers/operator_metrics_controller_test.go new file mode 100644 index 00000000000..7ead6afa433 --- /dev/null +++ b/controllers/operator_metrics_controller_test.go @@ -0,0 +1,365 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "gotest.tools/assert" + is "gotest.tools/assert/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const testOperatorNamespace = "openshift-gitops-operator" + +type fakeTokenRequester struct { + token string + expiry time.Time + err error +} + +func (f *fakeTokenRequester) RequestToken(_ context.Context, _, _ string, _ int64) (string, time.Time, error) { + if f.err != nil { + return "", time.Time{}, f.err + } + return f.token, f.expiry, nil +} + +func writeOperatorNamespaceFile(t *testing.T, namespace string) { + t.Helper() + dir := t.TempDir() + namespaceFile := filepath.Join(dir, "namespace") + if err := os.WriteFile(namespaceFile, []byte(namespace), 0o644); err != nil { + t.Fatal(err) + } + oldPath := operatorPodNamespacePath + operatorPodNamespacePath = namespaceFile + t.Cleanup(func() { + operatorPodNamespacePath = oldPath + }) +} + +func newOperatorMetricsTokenScheme() *runtime.Scheme { + s := scheme.Scheme + s.AddKnownTypes(monitoringv1.SchemeGroupVersion, &monitoringv1.ServiceMonitor{}) + return s +} + +func newOperatorMetricsServiceMonitor(namespace string, useLegacyAuth bool) *monitoringv1.ServiceMonitor { + sm := &monitoringv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorMetricsMonitorName, + Namespace: namespace, + }, + Spec: monitoringv1.ServiceMonitorSpec{ + Endpoints: []monitoringv1.Endpoint{ + { + Interval: monitoringv1.Duration("30s"), + Path: "/metrics", + Port: "metrics", + Scheme: "https", + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + ServerName: ptr.To("old-server-name"), + }, + }, + }, + }, + }, + } + if useLegacyAuth { + sm.Spec.Endpoints[0].BearerTokenSecret = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + } + } + return sm +} + +func TestGetOperatorNamespace_trimsNewline(t *testing.T) { + dir := t.TempDir() + namespaceFile := filepath.Join(dir, "namespace") + if err := os.WriteFile(namespaceFile, []byte("openshift-gitops-operator\n"), 0o644); err != nil { + t.Fatal(err) + } + oldPath := operatorPodNamespacePath + operatorPodNamespacePath = namespaceFile + t.Cleanup(func() { + operatorPodNamespacePath = oldPath + }) + + ns, err := getOperatorNamespace() + assert.NilError(t, err) + assert.Equal(t, ns, "openshift-gitops-operator") +} + +func TestBearerTokenRequeueDuration(t *testing.T) { + expiry := time.Now().Add(operatorMetricsTokenExpiry) + requeue := bearerTokenRequeueDuration(expiry) + assert.Assert(t, requeue > 19*time.Minute && requeue <= 20*time.Minute) + + // Cluster-capped shorter lifetime must still schedule renewal. + shortExpiry := time.Now().Add(30 * time.Minute) + shortRequeue := bearerTokenRequeueDuration(shortExpiry) + assert.Assert(t, shortRequeue > 9*time.Minute && shortRequeue <= 10*time.Minute) + + expiredExpiry := time.Now().Add(-time.Minute) + assert.Equal(t, bearerTokenRequeueDuration(expiredExpiry), time.Duration(0)) +} + +func TestOperatorMetricsTokenReconciler_migratesServiceMonitorAuth(t *testing.T) { + writeOperatorNamespaceFile(t, testOperatorNamespace) + + s := newOperatorMetricsTokenScheme() + serviceMonitor := newOperatorMetricsServiceMonitor(testOperatorNamespace, true) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(serviceMonitor).Build() + + r := &OperatorMetricsTokenReconciler{ + Client: c, + Scheme: s, + TokenRequester: &fakeTokenRequester{ + token: "test-token", + expiry: time.Now().Add(operatorMetricsTokenExpiry), + }, + } + + result, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: testOperatorNamespace, + }, + }) + assert.NilError(t, err) + assert.Assert(t, result.RequeueAfter > 0) + + updatedSM := &monitoringv1.ServiceMonitor{} + err = c.Get(context.Background(), types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: testOperatorNamespace, + }, updatedSM) + assert.NilError(t, err) + assert.Assert(t, is.Nil(updatedSM.Spec.Endpoints[0].BearerTokenSecret)) + assert.Assert(t, updatedSM.Spec.Endpoints[0].Authorization != nil) + assert.Equal(t, updatedSM.Spec.Endpoints[0].Authorization.Type, "Bearer") + assert.Equal(t, updatedSM.Spec.Endpoints[0].Authorization.Credentials.Name, operatorMetricsBearerTokenSecretName) + assert.Equal(t, *updatedSM.Spec.Endpoints[0].TLSConfig.ServerName, operatorMetricsServiceName+"."+testOperatorNamespace+".svc") +} + +func TestOperatorMetricsTokenReconciler_replacesLegacySecret(t *testing.T) { + writeOperatorNamespaceFile(t, testOperatorNamespace) + + s := newOperatorMetricsTokenScheme() + serviceMonitor := newOperatorMetricsServiceMonitor(testOperatorNamespace, false) + serviceMonitor.Spec.Endpoints[0].Authorization = &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + }, + } + legacySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, + Type: corev1.SecretTypeServiceAccountToken, + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(serviceMonitor, legacySecret).Build() + + expiry := time.Now().Add(operatorMetricsTokenExpiry) + r := &OperatorMetricsTokenReconciler{ + Client: c, + Scheme: s, + TokenRequester: &fakeTokenRequester{ + token: "new-token", + expiry: expiry, + }, + } + + _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: testOperatorNamespace, + }, + }) + assert.NilError(t, err) + + secret := &corev1.Secret{} + err = c.Get(context.Background(), types.NamespacedName{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, secret) + assert.NilError(t, err) + assert.Equal(t, secret.Type, corev1.SecretTypeOpaque) + assert.Equal(t, string(secret.Data[operatorMetricsBearerTokenKey]), "new-token") + parsedExpiry, err := parseBearerTokenExpiry(secret.Data[operatorMetricsBearerTokenExpiryKey]) + assert.NilError(t, err) + assert.Equal(t, parsedExpiry.UTC().Format(time.RFC3339), expiry.UTC().Format(time.RFC3339)) +} + +func TestOperatorMetricsTokenReconciler_keepsLegacySecretIfTokenRequestFails(t *testing.T) { + writeOperatorNamespaceFile(t, testOperatorNamespace) + + s := newOperatorMetricsTokenScheme() + serviceMonitor := newOperatorMetricsServiceMonitor(testOperatorNamespace, false) + serviceMonitor.Spec.Endpoints[0].Authorization = &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + }, + } + legacySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + operatorMetricsBearerTokenKey: []byte("legacy-token"), + }, + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(serviceMonitor, legacySecret).Build() + + r := &OperatorMetricsTokenReconciler{ + Client: c, + Scheme: s, + TokenRequester: &fakeTokenRequester{ + err: fmt.Errorf("token request failed"), + }, + } + + _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: testOperatorNamespace, + }, + }) + assert.ErrorContains(t, err, "token request failed") + + secret := &corev1.Secret{} + err = c.Get(context.Background(), types.NamespacedName{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, secret) + assert.NilError(t, err) + assert.Equal(t, secret.Type, corev1.SecretTypeServiceAccountToken) + assert.Equal(t, string(secret.Data[operatorMetricsBearerTokenKey]), "legacy-token") +} + +func TestOperatorMetricsTokenReconciler_refreshesExpiredToken(t *testing.T) { + writeOperatorNamespaceFile(t, testOperatorNamespace) + + s := newOperatorMetricsTokenScheme() + serviceMonitor := newOperatorMetricsServiceMonitor(testOperatorNamespace, false) + serviceMonitor.Spec.Endpoints[0].Authorization = &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: operatorMetricsBearerTokenSecretName, + }, + Key: operatorMetricsBearerTokenKey, + }, + } + expiredSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + operatorMetricsBearerTokenKey: []byte("old-token"), + operatorMetricsBearerTokenExpiryKey: []byte(time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)), + }, + } + c := fake.NewClientBuilder().WithScheme(s).WithObjects(serviceMonitor, expiredSecret).Build() + + newExpiry := time.Now().Add(operatorMetricsTokenExpiry) + r := &OperatorMetricsTokenReconciler{ + Client: c, + Scheme: s, + TokenRequester: &fakeTokenRequester{ + token: "refreshed-token", + expiry: newExpiry, + }, + } + + _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: testOperatorNamespace, + }, + }) + assert.NilError(t, err) + + secret := &corev1.Secret{} + err = c.Get(context.Background(), types.NamespacedName{ + Name: operatorMetricsBearerTokenSecretName, + Namespace: testOperatorNamespace, + }, secret) + assert.NilError(t, err) + assert.Equal(t, string(secret.Data[operatorMetricsBearerTokenKey]), "refreshed-token") +} + +func TestOperatorMetricsTokenReconciler_skipsOtherNamespaces(t *testing.T) { + writeOperatorNamespaceFile(t, testOperatorNamespace) + + s := newOperatorMetricsTokenScheme() + c := fake.NewClientBuilder().WithScheme(s).Build() + r := &OperatorMetricsTokenReconciler{ + Client: c, + Scheme: s, + TokenRequester: &fakeTokenRequester{ + token: "test-token", + expiry: time.Now().Add(operatorMetricsTokenExpiry), + }, + } + + result, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: operatorMetricsMonitorName, + Namespace: "other-namespace", + }, + }) + assert.NilError(t, err) + assert.Equal(t, result.RequeueAfter, time.Duration(0)) + + var secrets corev1.SecretList + err = c.List(context.Background(), &secrets, client.InNamespace("other-namespace")) + assert.NilError(t, err) + assert.Equal(t, len(secrets.Items), 0) +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index b39fcfb2d89..2ab324bb0a5 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -164,6 +164,12 @@ var _ = BeforeSuite(func() { }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + err = (&controllers.OperatorMetricsTokenReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + k8sClient, err := initK8sClient() Expect(err).ToNot(HaveOccurred()) diff --git a/test/nondefaulte2e/suite_test.go b/test/nondefaulte2e/suite_test.go index a0a4c14b473..46b3e7b8958 100644 --- a/test/nondefaulte2e/suite_test.go +++ b/test/nondefaulte2e/suite_test.go @@ -151,6 +151,12 @@ var _ = BeforeSuite(func() { }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + err = (&controllers.OperatorMetricsTokenReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + k8sClient, err := initK8sClient() Expect(err).ToNot(HaveOccurred()) diff --git a/test/openshift/e2e/ginkgo/parallel/1-104_validate_prometheus_alert_test.go b/test/openshift/e2e/ginkgo/parallel/1-104_validate_prometheus_alert_test.go index f2931d2b37a..2650f370f6a 100644 --- a/test/openshift/e2e/ginkgo/parallel/1-104_validate_prometheus_alert_test.go +++ b/test/openshift/e2e/ginkgo/parallel/1-104_validate_prometheus_alert_test.go @@ -1,6 +1,8 @@ package parallel import ( + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,11 +39,14 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() { Eventually(sm).Should(k8sFixture.ExistByName()) serverName := "openshift-gitops-operator-metrics-service.openshift-gitops-operator.svc" Expect(sm.Spec.Endpoints).To(Equal([]monitoringv1.Endpoint{{ - BearerTokenSecret: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "openshift-gitops-operator-metrics-monitor-bearer-token", + Authorization: &monitoringv1.SafeAuthorization{ + Type: "Bearer", + Credentials: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "openshift-gitops-operator-metrics-monitor-bearer-token", + }, + Key: "token", }, - Key: "token", }, Interval: monitoringv1.Duration("30s"), Path: "/metrics", @@ -68,6 +73,21 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() { "control-plane": "gitops-operator", }, })) + + bearerTokenSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "openshift-gitops-operator-metrics-monitor-bearer-token", + Namespace: "openshift-gitops-operator", + }, + } + Eventually(bearerTokenSecret).Should(k8sFixture.ExistByName()) + Expect(bearerTokenSecret.Type).To(Equal(corev1.SecretTypeOpaque)) + Expect(bearerTokenSecret.Data).To(HaveKey("token")) + Expect(bearerTokenSecret.Data).To(HaveKey("expiry")) + + expiry, err := time.Parse(time.RFC3339, string(bearerTokenSecret.Data["expiry"])) + Expect(err).NotTo(HaveOccurred()) + Expect(expiry.After(time.Now())).To(BeTrue()) }) }) })