Merge pull request #348 from detiber/ha_master

Ha master

15 files changed, 234 insertions, 22 deletions

diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example
index 41216dd89..56f4da5a2 100644
--- a/inventory/byo/hosts.example
+++ b/inventory/byo/hosts.example
@@ -38,14 +38,30 @@ deployment_type=enterprise
 # Allow all auth
 #openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}]
 
+# master cluster ha variables using pacemaker or RHEL HA
+#openshift_master_cluster_password=openshift_cluster
+#openshift_master_cluster_vip=192.168.133.25
+#openshift_master_cluster_public_vip=192.168.133.25
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+
+# master cluster ha variables when using a different HA solution
+# For installation the value of openshift_master_cluster_hostname must resolve
+# to the first master defined in the inventory.
+# The HA solution must be manually configured after installation and must ensure
+# that openshift-master is running on a single master host.
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_defer_ha=True
+
 # host group for masters
 [masters]
-ose3-master-ansible.test.example.com
+ose3-master[1:3]-ansible.test.example.com
 
 [etcd]
 ose3-etcd[1:3]-ansible.test.example.com
 
 # host group for nodes
 [nodes]
-ose3-master-ansible.test.example.com openshift_scheduleable=False
+ose3-master[1:3]-ansible.test.example.com openshift_scheduleable=False
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 3956128e1..904ad2dab 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -27,6 +27,9 @@
           api_url: "{{ openshift_master_api_url | default(None) }}"
           api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
           public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
+          cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+          cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+          cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
           console_path: "{{ openshift_master_console_path | default(None) }}"
           console_port: "{{ openshift_master_console_port | default(None) }}"
           console_url: "{{ openshift_master_console_url | default(None) }}"
@@ -152,16 +155,26 @@
   roles:
   - openshift_master_certificates
   post_tasks:
+  - name: Remove generated etcd client certs when using external etcd
+    file:
+      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+      state: absent
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    with_nested:
+    - masters_needing_certs
+    - - master.etcd-client.crt
+      - master.etcd-client.key
+
   - name: Create a tarball of the master certs
     command: >
-      tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz
-        -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} .
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
     args:
-      creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
     with_items: masters_needing_certs
   - name: Retrieve the master cert tarball from the master
     fetch:
-      src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
       dest: "{{ sync_tmpdir }}/"
       flat: yes
       fail_on_missing: yes
@@ -172,6 +185,7 @@
   hosts: oo_masters_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -192,9 +206,14 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Deploy OpenShift examples
+- name: Additional master configuration
   hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
   roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool
   - openshift_examples
 
 # Additional instance config for online deployments
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 727861b07..d733639c3 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -367,9 +367,11 @@ def set_url_facts_if_unset(facts):
         console_path = facts['master']['console_path']
         etcd_use_ssl = facts['master']['etcd_use_ssl']
         etcd_hosts = facts['master']['etcd_hosts']
-        etcd_port = facts['master']['etcd_port'],
+        etcd_port = facts['master']['etcd_port']
         hostname = facts['common']['hostname']
         public_hostname = facts['common']['public_hostname']
+        cluster_hostname = facts['master'].get('cluster_hostname')
+        cluster_public_hostname = facts['master'].get('cluster_public_hostname')
 
         if 'etcd_urls' not in facts['master']:
             etcd_urls = []
@@ -384,24 +386,51 @@ def set_url_facts_if_unset(facts):
                                         etcd_port)]
             facts['master']['etcd_urls'] = etcd_urls
         if 'api_url' not in facts['master']:
-            facts['master']['api_url'] = format_url(api_use_ssl, hostname,
+            api_hostname = cluster_hostname if cluster_hostname else hostname
+            facts['master']['api_url'] = format_url(api_use_ssl, api_hostname,
                                                     api_port)
         if 'public_api_url' not in facts['master']:
+            api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_api_url'] = format_url(api_use_ssl,
-                                                           public_hostname,
+                                                           api_public_hostname,
                                                            api_port)
         if 'console_url' not in facts['master']:
+            console_hostname = cluster_hostname if cluster_hostname else hostname
             facts['master']['console_url'] = format_url(console_use_ssl,
-                                                        hostname,
+                                                        console_hostname,
                                                         console_port,
                                                         console_path)
         if 'public_console_url' not in facts['master']:
+            console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_console_url'] = format_url(console_use_ssl,
-                                                               public_hostname,
+                                                               console_public_hostname,
                                                                console_port,
                                                                console_path)
     return facts
 
+def set_aggregate_facts(facts):
+    """ Set aggregate facts
+
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with aggregated facts
+    """
+    all_hostnames = set()
+    if 'common' in facts:
+        all_hostnames.add(facts['common']['hostname'])
+        all_hostnames.add(facts['common']['public_hostname'])
+
+        if 'master' in facts:
+            if 'cluster_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_hostname'])
+            if 'cluster_public_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_public_hostname'])
+
+        facts['common']['all_hostnames'] = list(all_hostnames)
+
+    return facts
+
 def set_sdn_facts_if_unset(facts):
     """ Set sdn facts if not already present in facts dict
 
@@ -675,6 +704,7 @@ class OpenShiftFacts(object):
         facts = set_identity_providers_if_unset(facts)
         facts = set_registry_url_if_unset(facts)
         facts = set_sdn_facts_if_unset(facts)
+        facts = set_aggregate_facts(facts)
         return dict(openshift=facts)
 
     def get_defaults(self, roles):
@@ -713,7 +743,7 @@ class OpenShiftFacts(object):
                           session_name='ssn', session_secrets_file='',
                           access_token_max_seconds=86400,
                           auth_token_max_seconds=500,
-                          oauth_grant_method='auto')
+                          oauth_grant_method='auto', cluster_defer_ha=False)
             defaults['master'] = master
 
         if 'node' in roles:
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 11195e83e..ca8860099 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -15,6 +15,12 @@ os_firewall_allow:
   port: 24224/tcp
 - service: Fluentd td-agent udp
   port: 24224/udp
+- service: pcsd
+  port: 2224/tcp
+- service: Corosync UDP
+  port: 5404/udp
+- service: Corosync UDP
+  port: 5405/udp
 os_firewall_deny:
 - service: OpenShift api http
   port: 8080/tcp
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index 6fd4dfb51..d57f9a4ea 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -1,3 +1,4 @@
 ---
 - name: restart openshift-master
   service: name=openshift-master state=restarted
+  when: not openshift_master_ha
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 02905f32d..bb1689e5f 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -8,6 +8,10 @@
     - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods
   when: openshift_master_oauth_grant_method is defined
 
+- fail:
+    msg: "openshift_master_cluster_password must be set for multi-master installations"
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
+
 - name: Install OpenShift Master package
   yum: pkg=openshift-master state=present
   register: install_result
@@ -16,6 +20,9 @@
   openshift_facts:
     role: master
     local_facts:
+      cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+      cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+      cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
       debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}"
       api_port: "{{ openshift_master_api_port | default(None) }}"
       api_url: "{{ openshift_master_api_url | default(None) }}"
@@ -114,12 +121,26 @@
 
 - name: Start and enable openshift-master
   service: name=openshift-master enabled=yes state=started
+  when: not openshift_master_ha
   register: start_result
 
 - name: pause to prevent service restart from interfering with bootstrapping
   pause: seconds=30
   when: start_result | changed
 
+- name: Install cluster packages
+  yum: pkg=pcs state=present
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+  register: install_result
+
+- name: Start and enable cluster service
+  service: name=pcsd enabled=yes state=started
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+
+- name: Set the cluster user password
+  shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster
+  when: install_result | changed
+
 - name: Create the OpenShift client config dir(s)
   file:
     path: "~{{ item }}/.kube"
diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml
index 8163ecd7f..03eb7e15f 100644
--- a/roles/openshift_master_ca/tasks/main.yml
+++ b/roles/openshift_master_ca/tasks/main.yml
@@ -14,7 +14,7 @@
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }}
+      --hostnames={{ openshift.common.all_hostnames | join(',') }}
       --master={{ openshift.master.api_url }}
       --public-master={{ openshift.master.public_api_url }}
       --cert-dir={{ openshift_master_config_dir }} --overwrite=false
diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml
index b5a3f8e40..297d53bcd 100644
--- a/roles/openshift_master_certificates/tasks/main.yml
+++ b/roles/openshift_master_certificates/tasks/main.yml
@@ -7,14 +7,20 @@
   with_items: masters_needing_certs
 
 - file:
-    src: "{{ openshift_master_ca_cert }}"
-    dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt"
-  with_items: masters_needing_certs
+    src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
+    dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+    state: hard
+  with_nested:
+  - masters_needing_certs
+  - - ca.crt
+    - ca.key
+    - ca.serial.txt
+
 
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }}
+      --hostnames={{ item.openshift.common.all_hostnames | join(',') }}
       --master={{ item.openshift.master.api_url }}
       --public-master={{ item.openshift.master.public_api_url }}
       --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
@@ -22,3 +28,5 @@
   args:
     creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt"
   with_items: masters_needing_certs
+
+
diff --git a/roles/openshift_master_certificates/vars/main.yml b/roles/openshift_master_certificates/vars/main.yml
index 6e577b13b..6214f7918 100644
--- a/roles/openshift_master_certificates/vars/main.yml
+++ b/roles/openshift_master_certificates/vars/main.yml
@@ -1,6 +1,3 @@
 ---
 openshift_generated_configs_dir: /etc/openshift/generated-configs
 openshift_master_config_dir: /etc/openshift/master
-openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
-openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
-openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"
diff --git a/roles/openshift_master_cluster/README.md b/roles/openshift_master_cluster/README.md
new file mode 100644
index 000000000..f150981fa
--- /dev/null
+++ b/roles/openshift_master_cluster/README.md
@@ -0,0 +1,34 @@
+OpenShift Master Cluster
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/openshift_master_cluster/meta/main.yml b/roles/openshift_master_cluster/meta/main.yml
new file mode 100644
index 000000000..f3236e850
--- /dev/null
+++ b/roles/openshift_master_cluster/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }
diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml
new file mode 100644
index 000000000..3510f9f62
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/configure.yml
@@ -0,0 +1,43 @@
+---
+- fail:
+    msg: This role requires that openshift_master_cluster_vip is set
+  when: openshift_master_cluster_vip is not defined or not openshift_master_cluster_vip
+- fail:
+    msg: This role requires that openshift_master_cluster_public_vip is set
+  when: openshift_master_cluster_public_vip is not defined or not openshift_master_cluster_public_vip
+
+- name: Authenticate to the cluster
+  command: pcs cluster auth -u hacluster -p {{ openshift_master_cluster_password }} {{ omc_cluster_hosts }}
+
+- name: Create the cluster
+  command: pcs cluster setup --name openshift_master {{ omc_cluster_hosts }}
+
+- name: Start the cluster
+  command: pcs cluster start --all
+
+- name: Enable the cluster on all nodes
+  command: pcs cluster enable --all
+
+- name: Set default resource stickiness
+  command: pcs resource defaults resource-stickiness=100
+
+- name: Add the cluster VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_vip }} --group openshift-master
+
+- name: Add the cluster public VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_public_vip }} --group openshift-master
+  when: openshift_master_cluster_public_vip != openshift_master_cluster_vip
+
+- name: Add the cluster openshift-master service resource
+  command: pcs resource create master systemd:openshift-master op start timeout=90s stop timeout=90s --group openshift-master
+
+- name: Disable stonith
+  command: pcs property set stonith-enabled=false
+
+# TODO: handle case where api port is not 8443
+- name: Wait for the clustered master service to be available
+  wait_for:
+    host: "{{ openshift_master_cluster_vip }}"
+    port: 8443
+    state: started
+    timeout: 300
diff --git a/roles/openshift_master_cluster/tasks/configure_deferred.yml b/roles/openshift_master_cluster/tasks/configure_deferred.yml
new file mode 100644
index 000000000..a80b6c5b4
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/configure_deferred.yml
@@ -0,0 +1,8 @@
+---
+- debug: msg="Deferring config"
+
+- name: Start and enable openshift-master
+  service:
+    name: openshift-master
+    state: started
+    enabled: yes
diff --git a/roles/openshift_master_cluster/tasks/main.yml b/roles/openshift_master_cluster/tasks/main.yml
new file mode 100644
index 000000000..315947183
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+- name: Test if cluster is already configured
+  command: pcs status
+  register: pcs_status
+  changed_when: false
+  failed_when: false
+  when: not openshift.master.cluster_defer_ha | bool
+
+- include: configure.yml
+  when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr"
+
+- include: configure_deferred.yml
+  when: openshift.master.cluster_defer_ha | bool
diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml
index 64a799dfb..c9f02aaf0 100644
--- a/roles/openshift_node_certificates/tasks/main.yml
+++ b/roles/openshift_node_certificates/tasks/main.yml
@@ -25,7 +25,7 @@
   command: >
     {{ openshift.common.admin_binary }} create-server-cert
       --cert=server.crt --key=server.key --overwrite=true
-      --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname]|unique|join(",") }}
+      --hostnames={{ openshift.common.all_hostnames |join(",") }}
       --signer-cert={{ openshift_master_ca_cert }}
       --signer-key={{ openshift_master_ca_key }}
       --signer-serial={{ openshift_master_ca_serial }}