Merge pull request #17 from shiftstack/support_ipv6_secondary_OSPRH-6486

Support IPv6 Secondary Interface in shiftstack-qa automation

Author: ekuris-redhat

collection/stages/roles/cleanup/tasks/cleanup_ipv6_secondary.yml

@@ -0,0 +1,37 @@
+---
+- name: Check if IPv6 secondary resources exist (via resources file)
+  ansible.builtin.include_vars:
+    file: "{{ resources_file }}"
+    name: registered_resources
+  ignore_errors: true  # noqa: ignore-errors
+  register: resources_load
+
+- name: Cleanup IPv6 secondary OpenStack resources
+  when:
+    - resources_load is succeeded
+    - registered_resources.ipv6_secondary_router_name is defined
+  block:
+    - name: Detach IPv6 secondary subnets from router
+      ansible.builtin.shell: >
+        openstack router remove subnet {{ registered_resources.ipv6_secondary_router_name }} {{ item }}
+      loop: "{{ registered_resources.ipv6_secondary_subnet_ids | default([]) }}"
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+      changed_when: true
+      ignore_errors: true  # noqa: ignore-errors
+
+    - name: Delete IPv6 secondary router
+      openstack.cloud.router:
+        cloud: "{{ user_cloud }}"
+        state: absent
+        name: "{{ registered_resources.ipv6_secondary_router_name }}"
+      ignore_errors: true  # noqa: ignore-errors
+
+    - name: Delete IPv6 secondary networks
+      ansible.builtin.shell: >
+        openstack network delete {{ item }}
+      loop: "{{ registered_resources.ipv6_secondary_network_ids | default([]) }}"
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+      changed_when: true
+      ignore_errors: true  # noqa: ignore-errors

collection/stages/roles/cleanup/tasks/main.yml

@@ -68,6 +68,10 @@
             - name: Discover the OpenShift installation type
               ansible.builtin.include_tasks: detect_ocp_installation.yml
 
+            - name: Cleanup IPv6 secondary network resources if they exist
+              ansible.builtin.include_tasks: cleanup_ipv6_secondary.yml
+              when: ocp_deployment_topology.secondary_ip_protocol | default('') == 'ipv6'
+
             - name: Destroy the OpenShift cluster if it exists
               ansible.builtin.include_tasks: destroy_openshift_cluster.yml
               when: existing_ocp_installation_type != ''

collection/stages/roles/day2ops/files/worker-machineconfig-dhcp.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 05-worker-kernelarg-dhcp
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+  kernelArguments:
+    - ip=dhcp,dhcp6

collection/stages/roles/day2ops/tasks/procedures/configure_ipv6_secondary.yml

@@ -0,0 +1,209 @@
+---
+- name: Apply DHCP MachineConfig to workers
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    definition: "{{ lookup('file', '../files/worker-machineconfig-dhcp.yaml') | from_yaml }}"
+
+- name: Wait for the MCP to finish the cluster updates
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_mcp_updated.yml
+  vars:
+    wait_retries: 60
+    wait_delay: 60
+
+- name: Active wait until all the ClusterOperators are ready
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_until_cluster_operators_ready.yml
+
+- name: Wait until OCP cluster is healthy
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_until_cluster_is_healthy.yml
+
+- name: Get OCP worker nodes
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: v1
+    kind: Node
+    label_selectors:
+      - node-role.kubernetes.io/worker
+  register: workers
+
+- name: Store the OCP worker node names
+  ansible.builtin.set_fact:
+    ocp_workers: "{{ workers | json_query('resources[*].metadata.name') | sort }}"
+    num_workers: "{{ workers.resources | length }}"
+
+- name: Discover IPv6 interfaces on all workers via live state
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    timeout 120 oc debug node/{{ item.0 }} -- chroot /host \
+    ip -6 addr show scope global | \
+    grep -B1 '{{ item.1.cidr | regex_replace('::/64$', '') }}' | \
+    awk -F': ' '/^[0-9]/{print $2}' | head -1
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  with_nested:
+    - "{{ ocp_workers }}"
+    - "{{ ipv6_secondary_networks.networks }}"
+  register: all_worker_interfaces
+  changed_when: false
+  retries: 10
+  delay: 30
+  until: all_worker_interfaces.stdout != ""
+
+- name: Validate consistent NIC naming across all workers
+  ansible.builtin.assert:
+    that:
+      - per_network_interfaces | unique | length == 1
+    fail_msg: |
+      NIC naming inconsistency for {{ item.net_name }}: {{ per_network_interfaces }}
+      Workers have different interface names for the same IPv6 network.
+  vars:
+    per_network_interfaces: "{{ all_worker_interfaces.results | selectattr('item.1.net_name', 'equalto', item.net_name) | map(attribute='stdout') | list }}"
+  loop: "{{ ipv6_secondary_networks.networks }}"
+
+- name: Store IPv6 interface names (validated consistent across workers)
+  ansible.builtin.set_fact:
+    ipv6_interfaces: "{{ all_worker_interfaces.results | selectattr('item.0', 'equalto', ocp_workers[0]) | map(attribute='stdout') | list }}"
+
+- name: Validate discovered IPv6 interfaces are not empty
+  ansible.builtin.assert:
+    that:
+      - ipv6_interfaces | length == ipv6_secondary_networks.networks | length
+      - ipv6_interfaces | select('equalto', '') | list | length == 0
+    fail_msg: |
+      Failed to discover IPv6 interfaces on workers.
+      Expected {{ ipv6_secondary_networks.networks | length }} interfaces, got: {{ ipv6_interfaces }}
+
+- name: Display discovered IPv6 interfaces
+  ansible.builtin.debug:
+    var: ipv6_interfaces
+
+- name: Template CNO macvlan patch
+  ansible.builtin.template:
+    src: network-macvlan.yml.j2
+    dest: "/tmp/network-macvlan.yml"
+    mode: u=rw,g=rw,o=r
+
+- name: Create OCP projects for IPv6 network testing
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: project.openshift.io/v1
+    kind: Project
+    name: "{{ item }}"
+    state: present
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Patch CNO with macvlan additionalNetworks
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc patch network.operator cluster --patch "$(cat /tmp/network-macvlan.yml)" --type merge
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: true
+
+- name: Verify NetworkAttachmentDefinition CR exists in {{ item }}
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: k8s.cni.cncf.io/v1
+    kind: NetworkAttachmentDefinition
+    namespace: "{{ item }}"
+  register: network_attachments
+  until:
+    - network_attachments is defined
+    - network_attachments is not failed
+    - network_attachments.resources | length > 0
+  retries: 10
+  delay: 30
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Template IPv6 test deployments
+  ansible.builtin.template:
+    src: ipv6-deployment.yml.j2
+    dest: "/tmp/ipv6-deployment-{{ item }}.yml"
+    mode: u=rw,g=rw,o=r
+  vars:
+    ipv6_project: "{{ item }}"
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Deploy hello-openshift pods with IPv6 network annotation
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    src: "/tmp/ipv6-deployment-{{ item }}.yml"
+    wait: true
+    wait_timeout: 300
+    namespace: "{{ item }}"
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Wait for pods to be ready in each IPv6 namespace
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ item }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: running_pods
+  until: running_pods.resources | length == num_workers | int
+  retries: 20
+  delay: 15
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Get IPv6 network IDs for port security operations
+  openstack.cloud.networks_info:
+    cloud: "{{ user_cloud }}"
+  register: all_openstack_networks
+
+- name: Build list of IPv6 secondary network IDs
+  ansible.builtin.set_fact:
+    ipv6_net_ids: "{{ all_openstack_networks.networks | selectattr('name', 'in', ipv6_secondary_networks.networks | map(attribute='net_name') | list) | map(attribute='id') | list }}"
+
+- name: Get worker ports on IPv6 secondary networks
+  openstack.cloud.port_info:
+    cloud: "{{ user_cloud }}"
+    filters:
+      device_owner: compute:nova
+  register: all_compute_ports
+
+- name: Filter worker ports on IPv6 networks
+  ansible.builtin.set_fact:
+    ipv6_worker_ports: "{{ all_compute_ports.ports | selectattr('network_id', 'in', ipv6_net_ids) | list }}"
+
+- name: Disable port security on worker IPv6 ports
+  openstack.cloud.port:
+    cloud: "{{ user_cloud }}"
+    state: present
+    name: "{{ item.id }}"
+    network: "{{ item.network_id }}"
+    port_security_enabled: false
+    no_security_groups: true
+  loop: "{{ ipv6_worker_ports }}"
+  loop_control:
+    label: "{{ item.id }}"
+
+- name: Verify pod-to-pod IPv6 connectivity on each network
+  ansible.builtin.include_tasks: procedures/verify_ipv6_connectivity.yml
+  loop: "{{ ipv6_secondary_networks.projects }}"
+  loop_control:
+    loop_var: ipv6_namespace
+
+- name: Verify external IPv6 reachability from worker nodes
+  ansible.builtin.include_tasks: procedures/verify_ipv6_external_reachability.yml
+  loop: "{{ ipv6_secondary_networks.projects }}"
+  loop_control:
+    loop_var: ipv6_namespace
+
+- name: Cleanup temporary manifest files
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop: "{{ ['/tmp/network-macvlan.yml'] + ipv6_secondary_networks.projects | map('regex_replace', '^(.*)$', '/tmp/ipv6-deployment-\\1.yml') | list }}"
+  loop_control:
+    label: "{{ item }}"
+  ignore_errors: true  # noqa: ignore-errors

collection/stages/roles/day2ops/tasks/procedures/verify_ipv6_connectivity.yml

@@ -0,0 +1,69 @@
+---
+- name: Reset pod IP mapping for {{ ipv6_namespace }}
+  ansible.builtin.set_fact:
+    ipv6_pod_ips: {}
+
+- name: Get pods in namespace {{ ipv6_namespace }}
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ ipv6_namespace }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: ipv6_pods
+
+- name: Store pod names for {{ ipv6_namespace }}
+  ansible.builtin.set_fact:
+    ipv6_pod_names: "{{ ipv6_pods.resources | map(attribute='metadata.name') | list }}"
+
+- name: Get pod IPv6 addresses on net1 interface
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc exec {{ item }} -n {{ ipv6_namespace }} -- ip -6 addr show dev net1 scope global | \
+    awk '/inet6/{print $2}' | cut -f1 -d'/'
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ipv6_pod_names }}"
+  register: pod_ipv6_results
+  changed_when: false
+  retries: 20
+  delay: 30
+  until: pod_ipv6_results.stdout != ""
+
+- name: Build pod name to IPv6 address mapping
+  ansible.builtin.set_fact:
+    ipv6_pod_ips: "{{ ipv6_pod_ips | default({}) | combine({item.item: item.stdout}) }}"
+  loop: "{{ pod_ipv6_results.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+
+- name: Display pod IPv6 addresses for {{ ipv6_namespace }}
+  ansible.builtin.debug:
+    var: ipv6_pod_ips
+
+- name: Validate all pods have IPv6 addresses
+  ansible.builtin.assert:
+    that:
+      - ipv6_pod_ips | length > 0
+      - ipv6_pod_ips.values() | select('equalto', '') | list | length == 0
+    fail_msg: |
+      Not all pods in {{ ipv6_namespace }} received IPv6 addresses.
+      Pod IPs: {{ ipv6_pod_ips }}
+
+- name: Check pod-to-pod IPv6 connectivity in {{ ipv6_namespace }}
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc exec {{ item[0] }} -n {{ ipv6_namespace }} -- /bin/curl -s --connect-timeout 10 http://[{{ item[1] }}]:8080
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  with_nested:
+    - "{{ ipv6_pod_ips.keys() | list }}"
+    - "{{ ipv6_pod_ips.values() | list }}"
+  when: ipv6_pod_ips[item[0]] != item[1]
+  register: connectivity_check
+  changed_when: false
+  retries: 5
+  delay: 10
+  until: connectivity_check.stdout is search('Hello OpenShift!')

collection/stages/roles/day2ops/tasks/procedures/verify_ipv6_external_reachability.yml

@@ -0,0 +1,50 @@
+---
+- name: Reset pod-node mapping for {{ ipv6_namespace }}
+  ansible.builtin.set_fact:
+    ipv6_ext_pod_nodes: {}
+
+- name: Get pods with node info in namespace {{ ipv6_namespace }} for external reachability test
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ ipv6_namespace }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: ipv6_ext_pods
+
+- name: Build pod-to-node mapping for {{ ipv6_namespace }}
+  ansible.builtin.set_fact:
+    ipv6_ext_pod_nodes: "{{ ipv6_ext_pod_nodes | default({}) | combine({item.metadata.name: item.spec.nodeName}) }}"
+  loop: "{{ ipv6_ext_pods.resources }}"
+  loop_control:
+    label: "{{ item.metadata.name }}"
+
+- name: Get pod IPv6 addresses for external test
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc exec {{ item }} -n {{ ipv6_namespace }} -- ip -6 addr show dev net1 scope global | \
+    awk '/inet6/{print $2}' | cut -f1 -d'/'
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ipv6_ext_pod_nodes.keys() | list }}"
+  register: ext_pod_ipv6_results
+  changed_when: false
+
+- name: Verify external IPv6 reachability from cross-node worker to pods in {{ ipv6_namespace }}
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    timeout 120 oc debug node/{{ ocp_workers | difference([ipv6_ext_pod_nodes[item.item]]) | first }} -- \
+    chroot /host curl -s --connect-timeout 10 http://[{{ item.stdout }}]:8080
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ext_pod_ipv6_results.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: item.stdout != ""
+  register: external_check
+  changed_when: false
+  retries: 5
+  delay: 10
+  until: external_check.stdout is search('Hello OpenShift!')