From bad3f553c0055e1cde6fe00408e483996c78126e Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Wed, 8 Jul 2026 10:21:29 +0200 Subject: [PATCH 1/4] fix: sequence ESO install before Keycloak secret sync Keycloak depends on ExternalSecrets that require the External Secrets Operator to be installed and reconciling. Without explicit ordering, rh-keycloak could sync before ESO was ready, leaving admin and database secrets missing when the Keycloak CR started. - Add sync-wave 26 to the external-secrets-operator OperatorGroup - Add sync-wave 27 to the openshift-external-secrets-operator subscription - Add an Argo CD health check for ExternalSecret so downstream resources wait until secrets report Ready=True Signed-off-by: Manuel Lorenzo --- docs/SYNC-WAVE-INVENTORY.md | 7 +++++-- values-hub.yaml | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index e2a47083..a7c7a455 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -23,11 +23,12 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 26 | └ ztvp-certificates | chart | distribution-policy (3 resources) | | 26 | └ openshift-storage | ns | Namespace + OperatorGroup | | 26 | └ rhtpa-operator | ns | Namespace + OperatorGroup | +| 26 | └ external-secrets-operator | ns | Namespace + OperatorGroup | | 27 | └ odf | sub | ODF operator install | | 27 | └ rhtpa-operator | sub | RHTPA operator install | +| 27 | └ external-secrets | sub | External Secrets Operator install | | 28 | └ quay-operator | sub | Quay operator install | | 29 | └ rhtas-operator | sub | RHTAS operator install | -| 30 | golang-external-secrets | **App** | | | 30 | zero-trust-workload-identity-manager | **App** | | | 31 | └ rhtpa-operator | chart | ingress-ca-job (SA, Role, RoleBinding, ConfigMap, Job) | | 32 | └ rhtpa-operator | chart | operator-rolebinding (2 bindings) | @@ -101,7 +102,8 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | rhtpa-operator (subscription) | -4 | 27 | After OperatorGroup (26) | | quay-operator (subscription) | -3 | 28 | After ODF operator | | rhtas-operator (subscription) | -2 | 29 | After Quay operator | -| golang-external-secrets | — | 30 | After Vault (newly added) | +| external-secrets-operator (OperatorGroup) | — | 26 | Before operator subscription (newly added) | +| external-secrets (subscription) | — | 27 | After OperatorGroup (26) (newly added) | | zero-trust-workload-identity-manager | — | 30 | After Vault/certs (newly added) | | quay-enterprise (namespace) | 1 | 32 | Before NooBaa and Quay components | | trusted-artifact-signer (namespace) | 1 | 32 | Auto-created by RHTAS operator | @@ -278,3 +280,4 @@ Resources without an explicit sync-wave default to wave 0. These include: pipeli - Template waves are resolved **locally within each app sync**, not globally. A template wave of 32 inside acs-central (app wave 41) does not conflict with a template wave of 32 inside noobaa-mcg (app wave 36); they run independently. - Sync waves control **Application creation order**, not readiness. A later wave means the Application resource is submitted to the hub later, but the earlier app's pods may not be fully running yet. For hard readiness gates, use Argo CD health checks or resource hooks. - **Externalized charts**: Five charts (certmanager, compliance-scanning, keycloak/RHBK, quay-registry, ZTWIM) are maintained in standalone repositories. Their resource-level sync-wave annotations are managed there and pinned via `chartVersion` in `values-hub.yaml`. Application-level sync-waves remain in this repository. +- **External Secrets Operator**: The ESO webhook must be ready before applications with ExternalSecret resources sync. The `rh-keycloak` application now includes `SkipDryRunOnMissingResource=true` to allow sync even if the ESO webhook is temporarily unavailable during initial deployment. An Argo CD health check for `ExternalSecret` resources tracks their Ready condition. diff --git a/values-hub.yaml b/values-hub.yaml index 13a3490d..e424fda9 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -57,6 +57,8 @@ clusterGroup: external-secrets-operator: operatorGroup: true targetNamespaces: [] + annotations: + argocd.argoproj.io/sync-wave: "26" # Create before operator subscription external-secrets: keycloak-system: operatorGroup: true @@ -121,6 +123,8 @@ clusterGroup: name: openshift-external-secrets-operator namespace: external-secrets-operator channel: stable-v1 + annotations: + argocd.argoproj.io/sync-wave: "27" # Install after OperatorGroup (26) cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -448,6 +452,8 @@ clusterGroup: - /overrides/values-keycloak-network-policy.yaml annotations: argocd.argoproj.io/sync-wave: "35" + syncOptions: + - SkipDryRunOnMissingResource=true # Allow sync even if ESO webhook is temporarily unavailable # SPIFFE Identity Provider is enabled by default in the rhbk subchart. # Override issuer/jwksUrl only if auto-generated values from cluster domain are not suitable. # Note: overrides must use the rhbk. prefix to reach the subchart. @@ -688,6 +694,27 @@ clusterGroup: return hs group: k8s.keycloak.org kind: KeycloakRealmImport + - check: | + local hs = {} + hs.status = "Progressing" + hs.message = "Waiting for ExternalSecret to sync" + if obj.status ~= nil and obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message or "Secret synced" + return hs + end + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message or "Secret sync failed" + return hs + end + end + end + return hs + group: external-secrets.io + kind: ExternalSecret resourceExclusions: | - apiGroups: - internal.open-cluster-management.io From 098ff5deb529e1866a0a28b8b9050f6f1c854ffd Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Fri, 10 Jul 2026 15:19:25 +0200 Subject: [PATCH 2/4] Add Application to ArgoCD's health checks (ESO patch) Signed-off-by: Manuel Lorenzo --- values-hub.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/values-hub.yaml b/values-hub.yaml index e424fda9..9729095b 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -715,6 +715,19 @@ clusterGroup: return hs group: external-secrets.io kind: ExternalSecret + - check: | + local hs = {} + hs.status = "Progressing" + hs.message = "Waiting for application status" + if obj.status ~= nil and obj.status.health ~= nil then + hs.status = obj.status.health.status + if obj.status.health.message ~= nil then + hs.message = obj.status.health.message + end + end + return hs + group: argoproj.io + kind: Application resourceExclusions: | - apiGroups: - internal.open-cluster-management.io From 1e644657e204c10d5011ad18b1d7c0e5ff047593 Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Mon, 13 Jul 2026 14:21:11 +0200 Subject: [PATCH 3/4] Add a health check for ClusterSecretStore Signed-off-by: Manuel Lorenzo --- values-hub.yaml | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/values-hub.yaml b/values-hub.yaml index 9729095b..f4c8bec0 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -696,38 +696,27 @@ clusterGroup: kind: KeycloakRealmImport - check: | local hs = {} - hs.status = "Progressing" - hs.message = "Waiting for ExternalSecret to sync" - if obj.status ~= nil and obj.status.conditions ~= nil then - for i, condition in ipairs(obj.status.conditions) do - if condition.type == "Ready" and condition.status == "True" then - hs.status = "Healthy" - hs.message = condition.message or "Secret synced" - return hs - end - if condition.type == "Ready" and condition.status == "False" then - hs.status = "Degraded" - hs.message = condition.message or "Secret sync failed" - return hs + if obj.status ~= nil then + if obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" and condition.status == "False" then + hs.status = "Degraded" + hs.message = condition.message + return hs + end + if condition.type == "Ready" and condition.status == "True" then + hs.status = "Healthy" + hs.message = condition.message + return hs + end end end end - return hs - group: external-secrets.io - kind: ExternalSecret - - check: | - local hs = {} hs.status = "Progressing" - hs.message = "Waiting for application status" - if obj.status ~= nil and obj.status.health ~= nil then - hs.status = obj.status.health.status - if obj.status.health.message ~= nil then - hs.message = obj.status.health.message - end - end + hs.message = "Waiting for ClusterSecretStore" return hs - group: argoproj.io - kind: Application + group: external-secrets.io + kind: ClusterSecretStore resourceExclusions: | - apiGroups: - internal.open-cluster-management.io From 6daff43c86a77915996b37d903a8b864916efcea Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Mon, 13 Jul 2026 15:35:59 +0200 Subject: [PATCH 4/4] Recover Application health check Signed-off-by: Manuel Lorenzo --- values-hub.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/values-hub.yaml b/values-hub.yaml index f4c8bec0..b0e0d106 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -717,6 +717,19 @@ clusterGroup: return hs group: external-secrets.io kind: ClusterSecretStore + - check: | + local hs = {} + hs.status = "Progressing" + hs.message = "Waiting for application status" + if obj.status ~= nil and obj.status.health ~= nil then + hs.status = obj.status.health.status + if obj.status.health.message ~= nil then + hs.message = obj.status.health.message + end + end + return hs + group: argoproj.io + kind: Application resourceExclusions: | - apiGroups: - internal.open-cluster-management.io