Wait for all nodes being added before running ansible

Openshift compute ndoes are members of AutoscalingGroup in heat
stack, when each member is created it registers itself on
bastion node ("deployment_bastion_node_add" SW deployment in
templates), then a separate step ("deployment_run_ansible"
in node.yaml) configures openshift on all nodes which are
already registered.

It may happen that deployment_run_ansible is trigerred by
some node before all nodes are registered (by deployment_bastion_node_add)
which causes that then nodes are added in multiple ansible
runs/batches. It doesn't break setup and basically is
nothing bad with it but it makes creation or scale up
slower. It would be better to wait until deployment_bastion_node_add
is done for all nodes and then run deployment_run_ansible
only once for all nodes (AutoscalingGroup doesn't allow to do this easily though).

This patch uses node_count param to get desired number of nodes,
this can not work with autoscaling though so it's used only for
not-autoscaling create/update scenario.

Author: jprovaznik

bastion.yaml

@@ -7,6 +7,11 @@ description: >
 
 parameters:
 
+  node_count:
+    type: number
+    description: >
+      Number of non-master nodes to create.
+
   # What version of OpenShift Container Platform to install
   # This value is used to select the RPM repo for the OCP release to install
   ocp_version:
@@ -376,6 +381,29 @@ resources:
           - get_file: templates/var/lib/ansible/roles/fstab_mount_options/tasks/main.yml
           - get_file: templates/var/lib/ansible/roles/xfs_grub_quota/tasks/main.yml
 
+  update_node_count:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: node_count
+      config: |
+        #!/bin/bash
+        set -eux
+        mkdir -p /var/lib/ansible
+        echo "$node_count" > /var/lib/ansible/node_count
+
+  deployment_update_node_count:
+    depends_on: wait_condition
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      config:
+        get_resource: update_node_count
+      server:
+        get_resource: host
+      input_values:
+        node_count: {get_param: node_count}
+
   deployment_write_templates:
     depends_on: wait_condition
     type: OS::Heat::SoftwareDeployment

fragments/bastion-ansible.sh

@@ -134,13 +134,27 @@ function is_scaleup() {
         grep -v '.*-node') && return 1 || return 0
 }
 
+function backup_ansdir() {
+    [ -e ${ANSDIR}.deployed ] && rm -rf ${ANSDIR}.deployed
+    mv ${ANSDIR}.started ${ANSDIR}.deployed
+}
+
 [ "$prepare_ansible" == "False" ] && exit 0
 
 mkdir -p /var/lib/ansible/group_vars
 mkdir -p /var/lib/ansible/host_vars
 
 touch $NODESFILE
 
+existing=$(wc -l < $NODESFILE)
+if [ -e /var/lib/ansible/node_count ]; then
+    node_count=$(cat /var/lib/ansible/node_count)
+    if [ $existing -lt $node_count -a "$autoscaling" != "True" ]; then
+        echo "skipping ansible run - only $existing of $node_count is registered"
+        exit 0
+    fi
+fi
+
 create_metadata_json /var/lib/ansible/metadata.json
 
 # generate ansible files from templates (located
@@ -184,6 +198,11 @@ export ANSIBLE_HOST_KEY_CHECKING=False
 
 logfile=/var/log/ansible.$$
 if is_scaleup; then
+    if [ -z $(get_new_nodes) ]; then
+        echo "There are no new nodes, not running scalup playbook"
+        backup_ansdir
+        exit 0
+    fi
     cmd="ansible-playbook -vvvv --inventory /var/lib/ansible/inventory \
         /var/lib/ansible/playbooks/scaleup.yml"
 else
@@ -197,8 +216,7 @@ if [ "$execute_ansible" == True ] ; then
         echo "Failed to run '$cmd', full log is in $(hostname):$logfile" >&2
         exit 1
     else
-        [ -e ${ANSDIR}.deployed ] && rm -rf ${ANSDIR}.deployed
-        mv ${ANSDIR}.started ${ANSDIR}.deployed
+        backup_ansdir
     fi
 else
     echo "INFO: ansible execution disabled"

node.yaml

@@ -346,6 +346,12 @@ parameters:
     type: string
     description: Extra parameters for openshift-ansible
 
+  autoscaling:
+    type: boolean
+    description: >
+      Automatically scale up/down openshift nodes.
+    default: false
+
 resources:
 
   # Generate a string to distinguish one node from the others
@@ -704,6 +710,8 @@ resources:
           default: {get_param: volume_quota}
         - name: extra_openshift_ansible_params
           default: {get_param: extra_openshift_ansible_params}
+        - name: autoscaling
+          default: {get_param: autoscaling}
       outputs:
         - name: ca_cert
         - name: ca_key

openshift.yaml

@@ -549,6 +549,7 @@ resources:
     depends_on: [external_router_interface, fixed_network, fixed_subnet, registry_volume]
     type: bastion.yaml
     properties:
+      node_count: {get_param: node_count}
       ocp_version: {get_param: ocp_version}
       osp_version: {get_param: osp_version}
       ansible_version: {get_param: ansible_version}
@@ -680,6 +681,7 @@ resources:
       resource:
         type: node.yaml
         properties:
+          autoscaling: {get_param: autoscaling}
           ocp_version: {get_param: ocp_version}
           image: {get_param: node_image}
           flavor: {get_param: node_flavor}