Validate pacemaker cluster members.

4 files changed, 54 insertions, 3 deletions

diff --git a/filter_plugins/openshift_master.py b/filter_plugins/openshift_master.py
index 8d7c62ad1..7ababc51a 100644
--- a/filter_plugins/openshift_master.py
+++ b/filter_plugins/openshift_master.py
@@ -463,6 +463,32 @@ class FilterModule(object):
         IdentityProviderBase.validate_idp_list(idp_list)
         return yaml.safe_dump([idp.to_dict() for idp in idp_list], default_flow_style=False)
 
+    @staticmethod
+    def validate_pcs_cluster(data, masters=None):
+        ''' Validates output from "pcs status", ensuring that each master
+            provided is online.
+            Ex: data = ('...',
+                        'PCSD Status:',
+                        'master1.example.com: Online',
+                        'master2.example.com: Online',
+                        'master3.example.com: Online',
+                        '...')
+                masters = ['master1.example.com',
+                           'master2.example.com',
+                           'master3.example.com']
+               returns True
+        '''
+        if not issubclass(type(data), str):
+            raise errors.AnsibleFilterError("|failed expects data is a string")
+        if not issubclass(type(masters), list):
+            raise errors.AnsibleFilterError("|failed expects masters is a list")
+        valid = True
+        for master in masters:
+            if "{0}: Online".format(master) not in data:
+                valid = False
+        return valid
+
     def filters(self):
         ''' returns a mapping of filters to methods '''
-        return {"translate_idps": self.translate_idps}
+        return {"translate_idps": self.translate_idps,
+                "validate_pcs_cluster": self.validate_pcs_cluster}
diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml
index 7603f0d61..fa13a64cb 100644
--- a/playbooks/common/openshift-master/restart.yml
+++ b/playbooks/common/openshift-master/restart.yml
@@ -73,6 +73,7 @@
     register: active_check_output
     when: openshift.master.cluster_method == 'pacemaker'
     failed_when: active_check_output.stdout not in ['active', 'inactive']
+    changed_when: false
   - set_fact:
       is_active: "{{ active_check_output.stdout == 'active' }}"
     when: openshift.master.cluster_method == 'pacemaker'
@@ -98,6 +99,18 @@
     with_items: "{{ groups.oo_masters_to_config | default([]) }}"
     when: (hostvars[item]['current_host'] | default(false)) | bool
 
+- name: Validate pacemaker cluster
+  hosts: oo_active_masters
+  tasks:
+  - name: Retrieve pcs status
+    command: pcs status
+    register: pcs_status_output
+    changed_when: false
+  - fail:
+      msg: >
+        Pacemaker cluster validation failed. One or more nodes are not online.
+    when: not (pcs_status_output.stdout | validate_pcs_cluster(groups.oo_masters_to_config)) | bool
+
 - name: Restart masters
   hosts: oo_masters_to_config:!oo_active_masters:!oo_current_masters
   vars:
diff --git a/playbooks/common/openshift-master/restart_hosts.yml b/playbooks/common/openshift-master/restart_hosts.yml
index 598e1ad63..ff206f5a2 100644
--- a/playbooks/common/openshift-master/restart_hosts.yml
+++ b/playbooks/common/openshift-master/restart_hosts.yml
@@ -16,8 +16,6 @@
       delay=10
       port="{{ openshift.master.api_port }}"
   when: openshift.master.cluster_method != 'pacemaker'
-# When cluster_method is pacemaker we can only ensure that the host
-# restarted successfully.
 - name: Wait for master to start
   become: no
   local_action:
@@ -25,4 +23,17 @@
       host="{{ inventory_hostname }}"
       state=started
       delay=10
+      port=22
   when: openshift.master.cluster_method == 'pacemaker'
+- name: Wait for master to become available
+  command: pcs status
+  register: pcs_status_output
+  until: pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname]) | bool
+  retries: 15
+  delay: 2
+  changed_when: false
+  when: openshift.master.cluster_method == 'pacemaker'
+- fail:
+    msg: >
+      Pacemaker cluster validation failed {{ inventory hostname }} is not online.
+  when: openshift.master.cluster_method == 'pacemaker' and not (pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname])) | bool
diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml
index 87fa99a3b..e40a1b329 100644
--- a/roles/openshift_facts/tasks/main.yml
+++ b/roles/openshift_facts/tasks/main.yml
@@ -10,6 +10,7 @@
   shell: ls /run/ostree-booted
   ignore_errors: yes
   failed_when: false
+  changed_when: false
   register: ostree_output
 
 # Locally setup containerized facts for now