[skmo] Add Skupper for cross-region RabbitMQ and Keystone internal routing

vakwetu · claude · vakwetu · commit 74abdbd27ff5 · 2026-04-07T18:56:32.000Z
Refactors Skupper integration to use a single generic skupper-listener.yaml
(parameterised via extra_vars) and a shared skupper-connector-tasks.yaml
included by service-specific connector wrappers.  Adds Keystone internal
endpoint routing via Skupper so the leaf region can reach the central
Keystone over the secure cross-namespace tunnel.

Key changes:
- skupper-listener.yaml: generic Listener playbook; cifmw_skupper_listener_*
  extra_vars control name/cert/port/routing-key; cifmw_skupper_listener_enabled
  flag skips all tasks when false (disabled case).
- skupper-connector-tasks.yaml: shared include-file that creates the Skupper
  Connector CR and waits for Configured status.
- skupper-connector.yaml (RabbitMQ): delegates connector create/wait to the
  shared task file.
- skupper-keystone-connector.yaml: sets keystoneInternalURL in skmo-values.yaml
  via lineinfile before kustomize runs; discovers the Keystone TLS secret from
  the KeystoneAPI CR; delegates connector create/wait to the shared task file.
  Uses keystone-internal.openstack.svc.cluster.local (the actual service name).
- skupper-keystone-listener.yaml: deleted; functionality merged into the generic
  skupper-listener.yaml invoked with Keystone-specific extra_vars.
- multi-namespace-skmo.yaml: Keystone Listener/Connector hooks moved to
  pre_stage_run (before OSCP deployment); RabbitMQ Listener hook converted to
  use the generic skupper-listener.yaml with extra_vars.

Signed-off-by: Ade Lee &lt;alee@redhat.com&gt;
Co-Authored-By: Claude &lt;noreply@anthropic.com&gt;
diff --git a/hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml b/hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml
@@ -0,0 +1,65 @@
+---
+# Patch the leaf/workload region OpenStackControlPlane to use the Skupper
+# Listener virtual endpoint for internal Keystone authentication traffic.
+#
+# The public endpoint override is left unchanged so that end-user traffic and
+# the Keystone service catalog continue to reference the central region's
+# external (public) URL.  Only the *internal* override — used for all
+# service-to-service communication inside the workload namespace — is updated
+# to point at the Skupper Listener.
+#
+# Run skupper-keystone-connector.yaml and skupper-keystone-listener.yaml
+# before this playbook so that the Skupper virtual service is in place.
+#
+# Variables:
+#   cifmw_skupper_leaf_namespace              (default: openstack2)
+#   cifmw_skupper_keystone_listener_host      (default: keystone-regionone)
+#   cifmw_skupper_keystone_port               (default: 5000)
+- name: Configure leaf region to use Skupper Keystone internal endpoint
+  hosts: localhost
+  gather_facts: false
+  vars:
+    cifmw_skupper_leaf_namespace: openstack2
+    cifmw_skupper_keystone_listener_host: keystone-regionone
+    cifmw_skupper_keystone_port: 5000
+  tasks:
+    - name: Build the Skupper Keystone internal URL
+      ansible.builtin.set_fact:
+        _skupper_keystone_internal_url: >-
+          https://{{ cifmw_skupper_keystone_listener_host }}.{{ cifmw_skupper_leaf_namespace }}.svc.cluster.local:{{ cifmw_skupper_keystone_port }}
+
+    - name: Patch leaf OSCP internal Keystone override to use Skupper endpoint
+      # This switches the internal keystone endpoint URL from the central
+      # region's public URL to the Skupper Listener virtual service.  The
+      # public endpoint override is not touched.
+      kubernetes.core.k8s:
+        state: patched
+        api_version: core.openstack.org/v1beta1
+        kind: OpenStackControlPlane
+        name: controlplane
+        namespace: "{{ cifmw_skupper_leaf_namespace }}"
+        definition:
+          spec:
+            keystone:
+              template:
+                override:
+                  service:
+                    internal:
+                      endpointURL: "{{ _skupper_keystone_internal_url }}"
+
+    - name: Wait for leaf OSCP to reconcile after Keystone endpoint change
+      kubernetes.core.k8s_info:
+        api_version: core.openstack.org/v1beta1
+        kind: OpenStackControlPlane
+        name: controlplane
+        namespace: "{{ cifmw_skupper_leaf_namespace }}"
+      register: _leaf_oscp
+      retries: 60
+      delay: 30
+      until:
+        - _leaf_oscp.resources | length > 0
+        - _leaf_oscp.resources[0].status is defined
+        - _leaf_oscp.resources[0].status.conditions is defined
+        - _leaf_oscp.resources[0].status.conditions |
+          selectattr('type', 'equalto', 'Ready') |
+          selectattr('status', 'equalto', 'True') | list | length > 0
diff --git a/hooks/playbooks/skmo/configure-leaf-listener.yaml b/hooks/playbooks/skmo/configure-leaf-listener.yaml
@@ -1,36 +1,64 @@
 ---
-- name: Patch leaf control plane with barbican-keystone-listener transport URL
+# Configure the leaf barbican-keystone-listener to use the Skupper
+# application network for cross-region RabbitMQ access.
+#
+# In the leaf region:
+#   - Read the RabbitMQ credentials from the dedicated user credentials secret
+#     created by the RabbitMQ operator when the TransportURL CR is reconciled.
+#   - Patch barbicanKeystoneListener to connect to the central RabbitMQ via the
+#     Skupper Listener endpoint using those credentials and its own pool_name.
+#
+# Variables:
+#   cifmw_skupper_central_namespace       (default: openstack)
+#   cifmw_skupper_leaf_namespace          (default: openstack2)
+#   cifmw_skupper_listener_host           (default: rabbitmq-regionone)
+#     Must match the host set in skupper-listener.yaml.
+#   cifmw_skupper_rabbitmq_port           (default: 5671)
+#   cifmw_skupper_transport_url_name      (default: barbican-keystone-listener-regiontwo)
+#     Name of the TransportURL CR created in prepare-leaf.yaml.  The operator
+#     creates a user credentials secret named:
+#     rabbitmq-user-<name>-<username>-user
+#   cifmw_skupper_transport_url_username  (default: barbican-keystone-listener-regiontwo)
+#     Must match the username field set on the TransportURL CR in prepare-leaf.yaml.
+- name: Configure barbican-keystone-listener to use Skupper for cross-region RabbitMQ
   hosts: localhost
   gather_facts: false
   vars:
-    central_namespace: openstack
-    leaf_namespace: openstack2
-    leaf_transport_url_name: rabbitmq-transport-url-barbican-keystone-listener-regiontwo
+    cifmw_skupper_central_namespace: openstack
+    cifmw_skupper_leaf_namespace: openstack2
+    cifmw_skupper_listener_host: rabbitmq-regionone
+    cifmw_skupper_rabbitmq_port: 5671
+    cifmw_skupper_transport_url_name: barbican-keystone-listener-regiontwo
+    cifmw_skupper_transport_url_username: barbican-keystone-listener-regiontwo
   tasks:
-    - name: Get transport URL secret from central namespace
+    - name: Get RabbitMQ user credentials secret for leaf listener
+      # The RabbitMQ operator creates a secret named
+      # rabbitmq-user-<transport-url-name>-<username>-user that contains
+      # the username and password fields for the dedicated RabbitMQ user.
       kubernetes.core.k8s_info:
         api_version: v1
         kind: Secret
-        namespace: "{{ central_namespace }}"
-        name: "{{ leaf_transport_url_name }}"
-      register: _transport_secret
+        namespace: "{{ cifmw_skupper_central_namespace }}"
+        name: "rabbitmq-user-{{ cifmw_skupper_transport_url_name }}-{{ cifmw_skupper_transport_url_username }}-user"
+      register: _rabbitmq_user_secret
 
-    - name: Patch OpenStackControlPlane in leaf region with notifications transport_url
+    - name: Patch leaf barbicanKeystoneListener to use Skupper RabbitMQ endpoint
       vars:
-        _transport_url: "{{ _transport_secret.resources[0].data['transport_url'] | b64decode }}"
+        _username: "{{ _rabbitmq_user_secret.resources[0].data['username'] | b64decode }}"
+        _password: "{{ _rabbitmq_user_secret.resources[0].data['password'] | b64decode }}"
       kubernetes.core.k8s:
         state: patched
         api_version: core.openstack.org/v1beta1
         kind: OpenStackControlPlane
         name: controlplane
-        namespace: "{{ leaf_namespace }}"
+        namespace: "{{ cifmw_skupper_leaf_namespace }}"
         definition:
           spec:
             barbican:
               template:
                 barbicanKeystoneListener:
                   customServiceConfig: |
                     [DEFAULT]
-                    transport_url = {{ _transport_url }}
+                    transport_url = rabbit://{{ _username }}:{{ _password }}@{{ cifmw_skupper_listener_host }}:{{ cifmw_skupper_rabbitmq_port }}/?ssl=1
                     [keystone_notifications]
-                    pool_name = barbican-listener-regionTwo
+                    pool_name = barbican-listener-regiontwo
diff --git a/hooks/playbooks/skmo/prepare-leaf.yaml b/hooks/playbooks/skmo/prepare-leaf.yaml
@@ -10,8 +10,8 @@
     central_rootca_secret: rootca-public
     central_rootca_internal_secret: rootca-internal
     leaf_transport_url_name: barbican-keystone-listener-regiontwo
+    leaf_transport_url_username: barbican-keystone-listener-regiontwo
     leaf_transport_url_name_secret: rabbitmq-transport-url-barbican-keystone-listener-regiontwo
-    leaf_transport_url_secret_copy: barbican-keystone-listener-regiontwo-transport
   tasks:
     - name: Wait for central Keystone API to be ready
       kubernetes.core.k8s_info:
@@ -159,6 +159,7 @@
             namespace: "{{ central_namespace }}"
           spec:
             rabbitmqClusterName: rabbitmq
+            username: "{{ leaf_transport_url_username }}"
 
     - name: Wait for TransportURL to be ready
       kubernetes.core.k8s_info:
@@ -176,23 +177,3 @@
         - _transport_url_info.resources[0].status.conditions |
           selectattr('type', 'equalto', 'Ready') |
           selectattr('status', 'equalto', 'True') | list | length > 0
-
-    - name: Get transport URL secret from central namespace
-      kubernetes.core.k8s_info:
-        api_version: v1
-        kind: Secret
-        namespace: "{{ central_namespace }}"
-        name: "{{ leaf_transport_url_name_secret }}"
-      register: _transport_secret
-
-    - name: Copy transport URL secret to leaf namespace
-      kubernetes.core.k8s:
-        state: present
-        definition:
-          apiVersion: v1
-          kind: Secret
-          metadata:
-            name: "{{ leaf_transport_url_secret_copy }}"
-            namespace: "{{ leaf_namespace }}"
-          type: "{{ _transport_secret.resources[0].type }}"
-          data: "{{ _transport_secret.resources[0].data }}"
diff --git a/hooks/playbooks/skmo/skupper-connector-tasks.yaml b/hooks/playbooks/skmo/skupper-connector-tasks.yaml
@@ -0,0 +1,51 @@
+---
+# Shared task file: Create a Skupper Connector CR and wait for it to be
+# Configured.  Include this from service-specific connector playbooks after
+# the TLS credentials have been discovered and stored in the variables below.
+#
+# Expected variables (set via include_tasks vars: block):
+#   _cifmw_connector_name              Skupper Connector CR name
+#   _cifmw_connector_namespace         Namespace for the Connector
+#   _cifmw_connector_routing_key       Skupper routing key (must match Listener)
+#   _cifmw_connector_host              Backend service hostname
+#   _cifmw_connector_port              Backend service port
+#   _cifmw_connector_tls_credentials   Name of the TLS Secret for the backend
+#   _cifmw_connector_verify_hostname   Whether Skupper verifies the backend cert hostname
+#   _cifmw_connector_ignore_wait_errors  Whether to ignore wait failures
+
+- name: Create Skupper Connector
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      apiVersion: skupper.io/v2alpha1
+      kind: Connector
+      metadata:
+        name: "{{ _cifmw_connector_name }}"
+        namespace: "{{ _cifmw_connector_namespace }}"
+      spec:
+        routingKey: "{{ _cifmw_connector_routing_key }}"
+        host: "{{ _cifmw_connector_host }}"
+        port: "{{ _cifmw_connector_port }}"
+        type: tcp
+        tlsCredentials: "{{ _cifmw_connector_tls_credentials }}"
+        verifyHostname: "{{ _cifmw_connector_verify_hostname }}"
+
+- name: Wait for Skupper Connector to be configured
+  # A Connector shows "Ready" only after a matching Listener is deployed in
+  # the remote namespace.  Waiting for "Configured" is sufficient here.
+  ignore_errors: "{{ _cifmw_connector_ignore_wait_errors | bool }}"
+  kubernetes.core.k8s_info:
+    api_version: skupper.io/v2alpha1
+    kind: Connector
+    name: "{{ _cifmw_connector_name }}"
+    namespace: "{{ _cifmw_connector_namespace }}"
+  register: _connector
+  retries: 30
+  delay: 10
+  until:
+    - _connector.resources | length > 0
+    - _connector.resources[0].status is defined
+    - _connector.resources[0].status.conditions is defined
+    - _connector.resources[0].status.conditions |
+      selectattr('type', 'equalto', 'Configured') |
+      selectattr('status', 'equalto', 'True') | list | length > 0
diff --git a/hooks/playbooks/skmo/skupper-connector.yaml b/hooks/playbooks/skmo/skupper-connector.yaml
@@ -0,0 +1,39 @@
+---
+# Create a Skupper Connector in the central namespace that exposes the central
+# RabbitMQ service to workload regions over the Skupper application network.
+#
+# The TLS Secret name is auto-discovered from the RabbitmqCluster CR.
+# Connector creation and the wait for Configured status are handled by the
+# shared skupper-connector-tasks.yaml task file.
+#
+# Variables:
+#   cifmw_skupper_central_namespace    (default: openstack)
+#   cifmw_skupper_routing_key          (default: rabbit-keystone)
+#   cifmw_skupper_rabbitmq_port        (default: 5671)
+- name: Create Skupper Connector for central RabbitMQ
+  hosts: localhost
+  gather_facts: false
+  vars:
+    cifmw_skupper_central_namespace: openstack
+    cifmw_skupper_routing_key: rabbit-keystone
+    cifmw_skupper_rabbitmq_port: 5671
+  tasks:
+    - name: Get RabbitMQ TLS certificate secret name from RabbitmqCluster CR
+      kubernetes.core.k8s_info:
+        api_version: rabbitmq.com/v1beta1
+        kind: RabbitmqCluster
+        name: rabbitmq
+        namespace: "{{ cifmw_skupper_central_namespace }}"
+      register: _rabbitmq_cluster
+
+    - name: Create Skupper Connector and wait for Configured
+      ansible.builtin.include_tasks: skupper-connector-tasks.yaml
+      vars:
+        _cifmw_connector_name: rabbitmq-keystone
+        _cifmw_connector_namespace: "{{ cifmw_skupper_central_namespace }}"
+        _cifmw_connector_routing_key: "{{ cifmw_skupper_routing_key }}"
+        _cifmw_connector_host: "rabbitmq.{{ cifmw_skupper_central_namespace }}.svc.cluster.local"
+        _cifmw_connector_port: "{{ cifmw_skupper_rabbitmq_port }}"
+        _cifmw_connector_tls_credentials: "{{ _rabbitmq_cluster.resources[0].spec.tls.secretName }}"
+        _cifmw_connector_verify_hostname: true
+        _cifmw_connector_ignore_wait_errors: false
diff --git a/hooks/playbooks/skmo/skupper-install.yaml b/hooks/playbooks/skmo/skupper-install.yaml
@@ -0,0 +1,66 @@
+---
+# Install the Skupper operator (cluster-scope) on the current cluster.
+#
+# Variables:
+#   cifmw_skupper_install_source   (default: upstream)
+#     upstream   - apply the upstream install YAML from skupper.io
+#     downstream - apply a locally-downloaded Red Hat Service Interconnect YAML
+#   cifmw_skupper_upstream_install_url (default: https://skupper.io/v2/install.yaml)
+#   cifmw_skupper_downstream_install_file (no default; required when source=downstream)
+- name: Install Skupper operator
+  hosts: localhost
+  gather_facts: false
+  vars:
+    cifmw_skupper_install_source: upstream
+    cifmw_skupper_upstream_install_url: "https://skupper.io/v2/install.yaml"
+    cifmw_skupper_downstream_install_file: ""
+  tasks:
+    - name: Check if Skupper CRD is already present
+      kubernetes.core.k8s_info:
+        api_version: apiextensions.k8s.io/v1
+        kind: CustomResourceDefinition
+        name: sites.skupper.io
+      register: _skupper_crd
+
+    - name: Apply upstream Skupper install YAML
+      when:
+        - _skupper_crd.resources | length == 0
+        - cifmw_skupper_install_source == 'upstream'
+      ansible.builtin.command:
+        cmd: "oc apply -f {{ cifmw_skupper_upstream_install_url }}"
+
+    - name: Apply downstream Skupper install YAML (Red Hat Service Interconnect)
+      when:
+        - _skupper_crd.resources | length == 0
+        - cifmw_skupper_install_source == 'downstream'
+      ansible.builtin.assert:
+        that: cifmw_skupper_downstream_install_file | length > 0
+        fail_msg: >-
+          cifmw_skupper_downstream_install_file must be set when
+          cifmw_skupper_install_source is 'downstream'.
+
+    - name: Apply downstream install file
+      when:
+        - _skupper_crd.resources | length == 0
+        - cifmw_skupper_install_source == 'downstream'
+        - cifmw_skupper_downstream_install_file | length > 0
+      ansible.builtin.command:
+        cmd: "oc apply -f {{ cifmw_skupper_downstream_install_file }}"
+
+    - name: Wait for Skupper controller to be ready
+      # The upstream install names the deployment "skupper-controller" while the
+      # downstream Red Hat Service Interconnect install uses
+      # "skupper-controller-manager".  List all Deployments in the namespace and
+      # wait until at least one is fully ready, regardless of name.
+      kubernetes.core.k8s_info:
+        api_version: apps/v1
+        kind: Deployment
+        namespace: skupper
+      register: _skupper_deploy
+      retries: 30
+      delay: 10
+      until:
+        - _skupper_deploy.resources | length > 0
+        - _skupper_deploy.resources |
+          selectattr('status.readyReplicas', 'defined') |
+          selectattr('status.readyReplicas', 'ge', 1) | list | length > 0
diff --git a/hooks/playbooks/skmo/skupper-keystone-connector.yaml b/hooks/playbooks/skmo/skupper-keystone-connector.yaml
diff --git a/hooks/playbooks/skmo/skupper-listener.yaml b/hooks/playbooks/skmo/skupper-listener.yaml
diff --git a/hooks/playbooks/skmo/skupper-sites.yaml b/hooks/playbooks/skmo/skupper-sites.yaml
