Merge pull request #1337 from abutcher/master-scaleup

Support for adding new masters

21 files changed, 368 insertions, 131 deletions

diff --git a/playbooks/aws/openshift-cluster/cluster_hosts.yml b/playbooks/aws/openshift-cluster/cluster_hosts.yml
index ca87bc655..9a3361919 100644
--- a/playbooks/aws/openshift-cluster/cluster_hosts.yml
+++ b/playbooks/aws/openshift-cluster/cluster_hosts.yml
@@ -10,8 +10,12 @@ g_nfs_hosts:     "{{ g_all_hosts | intersect(groups['tag_host-type_nfs'] | defau
 
 g_master_hosts:  "{{ g_all_hosts | intersect(groups['tag_host-type_master'] | default([])) }}"
 
+g_new_master_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type_new_master'] | default([])) }}"
+
 g_node_hosts:    "{{ g_all_hosts | intersect(groups['tag_host-type_node'] | default([])) }}"
 
-g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type_infra'] | default([])) }}"
+g_new_node_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type_new_node'] | default([])) }}"
+
+g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type_infra']) | default([]) }}"
 
 g_compute_hosts: "{{ g_node_hosts | intersect(groups['tag_sub-host-type_compute'] | default([])) }}"
diff --git a/playbooks/byo/openshift-cluster/cluster_hosts.yml b/playbooks/byo/openshift-cluster/cluster_hosts.yml
index 10872e738..8893db245 100644
--- a/playbooks/byo/openshift-cluster/cluster_hosts.yml
+++ b/playbooks/byo/openshift-cluster/cluster_hosts.yml
@@ -5,6 +5,8 @@ g_lb_hosts:     "{{ groups.lb | default([]) }}"
 
 g_master_hosts: "{{ groups.masters | default([]) }}"
 
+g_new_master_hosts: "{{ groups.new_masters | default([]) }}"
+
 g_node_hosts:   "{{ groups.nodes | default([]) }}"
 
 g_new_node_hosts: "{{ groups.new_nodes | default([]) }}"
diff --git a/playbooks/byo/openshift-cluster/scaleup.yml b/playbooks/byo/openshift-master/scaleup.yml
index 1702690f6..18797d02a 100644
--- a/playbooks/byo/openshift-cluster/scaleup.yml
+++ b/playbooks/byo/openshift-master/scaleup.yml
@@ -1,5 +1,5 @@
 ---
-- include: ../../common/openshift-cluster/scaleup.yml
+- include: ../../common/openshift-master/scaleup.yml
   vars_files:
   - ../../byo/openshift-cluster/cluster_hosts.yml
   vars:
diff --git a/playbooks/byo/openshift-node/filter_plugins b/playbooks/byo/openshift-node/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/byo/openshift-node/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/byo/openshift-node/lookup_plugins b/playbooks/byo/openshift-node/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/byo/openshift-node/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/byo/openshift-node/roles b/playbooks/byo/openshift-node/roles
new file mode 120000
index 000000000..20c4c58cf
--- /dev/null
+++ b/playbooks/byo/openshift-node/roles
@@ -0,0 +1 @@
+../../../roles
\ No newline at end of file
diff --git a/playbooks/byo/openshift-node/scaleup.yml b/playbooks/byo/openshift-node/scaleup.yml
new file mode 100644
index 000000000..0343597b5
--- /dev/null
+++ b/playbooks/byo/openshift-node/scaleup.yml
@@ -0,0 +1,8 @@
+---
+- include: ../../common/openshift-node/scaleup.yml
+  vars_files:
+  - ../../byo/openshift-cluster/cluster_hosts.yml
+  vars:
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_debug_level: "{{ debug_level | default(2) }}"
+    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/common/openshift-cluster/additional_config.yml b/playbooks/common/openshift-cluster/additional_config.yml
new file mode 100644
index 000000000..0f4e6ab88
--- /dev/null
+++ b/playbooks/common/openshift-cluster/additional_config.yml
@@ -0,0 +1,55 @@
+- name: Configure flannel
+  hosts: oo_first_master
+  vars:
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+  roles:
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+
+- name: Additional master configuration
+  hosts: oo_first_master
+  vars:
+    cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+    openshift_master_ha: "{{ groups.oo_masters | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters | join(' ')}}"
+  roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker"
+  - role: openshift_examples
+    when: openshift.common.install_examples | bool
+  - role: openshift_cluster_metrics
+    when: openshift.common.use_cluster_metrics | bool
+  - role: openshift_manageiq
+    when: openshift.common.use_manageiq | bool
+  - role: cockpit
+    when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
+      (osm_use_cockpit | bool or osm_use_cockpit is undefined )
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+  - role: pods
+    when: openshift.common.deployment_type == 'online'
+  - role: os_env_extras
+    when: openshift.common.deployment_type == 'online'
+
+- name: Create persistent volumes and create hosted services
+  hosts: oo_first_master
+  vars:
+    attach_registry_volume: "{{ openshift.hosted.registry.storage.kind != None }}"
+    deploy_infra: "{{ openshift.master.infra_nodes | default([]) | length > 0 }}"
+    persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups) }}"
+    persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims }}"
+  roles:
+  - role: openshift_persistent_volumes
+    when: persistent_volumes | length > 0 or persistent_volume_claims | length > 0
+  - role: openshift_serviceaccounts
+    openshift_serviceaccounts_names:
+    - router
+    - registry
+    openshift_serviceaccounts_namespace: default
+    openshift_serviceaccounts_sccs:
+    - privileged
+  - role: openshift_router
+    when: deploy_infra | bool
+  - role: openshift_registry
+    when: deploy_infra | bool and attach_registry_volume | bool
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index 2cad4b362..23c8f039e 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -11,4 +11,6 @@
 
 - include: ../openshift-master/config.yml
 
+- include: additional_config.yml
+
 - include: ../openshift-node/config.yml
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index 7917bfba5..432a92b49 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -10,8 +10,8 @@
     when: g_etcd_hosts is not defined
 
   - fail:
-      msg: This playbook requires g_master_hosts to be set
-    when: g_master_hosts is not defined
+      msg: This playbook requires g_master_hosts or g_new_master_hosts to be set
+    when: g_master_hosts is not defined and g_new_master_hosts is not defined
 
   - fail:
       msg: This playbook requires g_node_hosts or g_new_node_hosts to be set
@@ -29,6 +29,14 @@
       msg: The nfs group must be limited to one host
     when: (groups[g_nfs_hosts] | default([])) | length > 1
 
+  - name: Evaluate oo_masters
+    add_host:
+      name: "{{ item }}"
+      groups: oo_masters
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_master_hosts | union(g_new_master_hosts) | default([]) }}"
+
   - name: Evaluate oo_etcd_to_config
     add_host:
       name: "{{ item }}"
@@ -43,11 +51,7 @@
       groups: oo_masters_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_master_hosts | default([]) }}"
-
-  # Use g_new_node_hosts if it exists otherwise g_node_hosts
-  - set_fact:
-      g_node_hosts_to_config: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
+    with_items: "{{ g_new_master_hosts | default(g_master_hosts | default([], true), true) }}"
 
   - name: Evaluate oo_nodes_to_config
     add_host:
@@ -55,7 +59,7 @@
       groups: oo_nodes_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_node_hosts_to_config | default([]) }}"
+    with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
 
   # Skip adding the master to oo_nodes_to_config when g_new_node_hosts is
   - name: Evaluate oo_nodes_to_config
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 77edbd1a6..b9d595576 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -57,6 +57,8 @@
           console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
           public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
           portal_net: "{{ openshift_master_portal_net | default(None) }}"
+          ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}"
+          master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}"
   - openshift_facts:
       role: hosted
       openshift_env:
@@ -144,7 +146,7 @@
     when: etcd_client_certs_missing is defined and etcd_client_certs_missing
 
 - name: Determine if master certificates need to be generated
-  hosts: oo_masters_to_config
+  hosts: oo_first_master:oo_masters_to_config
   tasks:
   - set_fact:
       openshift_master_certs_no_etcd:
@@ -221,15 +223,6 @@
       validate_checksum: yes
     with_items: masters_needing_certs
 
-- name: Compute haproxy_backend_servers
-  hosts: localhost
-  connection: local
-  become: no
-  gather_facts: no
-  tasks:
-  - set_fact:
-      haproxy_backend_servers: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_haproxy_backend_masters }}"
-
 - name: Configure load balancers
   hosts: oo_lb_to_config
   vars:
@@ -248,11 +241,11 @@
       mode: tcp
       option: tcplog
       balance: source
-      servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
+      servers: "{{ hostvars | oo_select_keys(groups['oo_masters']) | oo_haproxy_backend_masters }}"
   roles:
   - role: openshift_facts
   - role: haproxy
-    when: groups.oo_masters_to_config | length > 1
+    when: hostvars[groups.oo_first_master.0].openshift.master.ha | bool
 
 - name: Check for cached session secrets
   hosts: oo_first_master
@@ -334,8 +327,8 @@
   serial: 1
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
+    openshift_master_ha: "{{ openshift.master.ha }}"
+    openshift_master_count: "{{ openshift.master.master_count }}"
     openshift_master_session_auth_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_auth_secrets }}"
     openshift_master_session_encryption_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_encryption_secrets }}"
   pre_tasks:
@@ -343,12 +336,12 @@
     file:
       path: "{{ openshift.common.config_base }}/master"
       state: directory
-    when: master_certs_missing and 'oo_first_master' not in group_names
+    when: master_certs_missing | bool and 'oo_first_master' not in group_names
   - name: Unarchive the tarball on the master
     unarchive:
       src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
       dest: "{{ master_cert_config_dir }}"
-    when: master_certs_missing and 'oo_first_master' not in group_names
+    when: master_certs_missing | bool and 'oo_first_master' not in group_names
   roles:
   - openshift_master
   - role: nickhammond.logrotate
@@ -359,32 +352,12 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Additional master configuration
-  hosts: oo_first_master
-  vars:
-    cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
-    etcd_urls: "{{ openshift.master.etcd_urls }}"
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
+# Additional instance config for online deployments
+- name: Additional instance config
+  hosts: oo_masters_deployment_type_online
   roles:
-  - role: openshift_master_cluster
-    when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker"
-  - role: openshift_examples
-    when: openshift.common.install_examples | bool
-  - role: openshift_cluster_metrics
-    when: openshift.common.use_cluster_metrics | bool
-  - role: openshift_manageiq
-    when: openshift.common.use_manageiq | bool
-  - role: cockpit
-    when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
-      (osm_use_cockpit | bool or osm_use_cockpit is undefined )
-  - role: flannel_register
-    when: openshift.common.use_flannel | bool
-  - role: pods
-    when: openshift.common.deployment_type == 'online'
-  - role: os_env_extras
-    when: openshift.common.deployment_type == 'online'
-
+  - pods
+  - os_env_extras
 
 - name: Delete temporary directory on localhost
   hosts: localhost
@@ -394,26 +367,3 @@
   tasks:
   - file: name={{ g_master_mktemp.stdout }} state=absent
     changed_when: False
-
-- name: Create persistent volumes and create hosted services
-  hosts: oo_first_master
-  vars:
-    attach_registry_volume: "{{ openshift.hosted.registry.storage.kind != None }}"
-    deploy_infra: "{{ openshift.master.infra_nodes | default([]) | length > 0 }}"
-    persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups) }}"
-    persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims }}"
-  roles:
-  - role: openshift_persistent_volumes
-    when: persistent_volumes | length > 0 or persistent_volume_claims | length > 0
-  - role: openshift_serviceaccounts
-    openshift_serviceaccounts_names:
-    - router
-    - registry
-    openshift_serviceaccounts_namespace: default
-    openshift_serviceaccounts_sccs:
-    - privileged
-  - role: openshift_router
-    when: deploy_infra | bool
-  - role: openshift_registry
-    when: deploy_infra | bool and attach_registry_volume | bool
-
diff --git a/playbooks/common/openshift-master/library/modify_yaml.py b/playbooks/common/openshift-master/library/modify_yaml.py
new file mode 100755
index 000000000..a4be10ca3
--- /dev/null
+++ b/playbooks/common/openshift-master/library/modify_yaml.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vim: expandtab:tabstop=4:shiftwidth=4
+
+''' modify_yaml ansible module '''
+
+import yaml
+
+DOCUMENTATION = '''
+---
+module: modify_yaml
+short_description: Modify yaml key value pairs
+author: Andrew Butcher
+requirements: [ ]
+'''
+EXAMPLES = '''
+- modify_yaml:
+    dest: /etc/origin/master/master-config.yaml
+    yaml_key: 'kubernetesMasterConfig.masterCount'
+    yaml_value: 2
+'''
+
+def main():
+    ''' Modify key (supplied in jinja2 dot notation) in yaml file, setting
+        the key to the desired value.
+    '''
+
+    # disabling pylint errors for global-variable-undefined and invalid-name
+    # for 'global module' usage, since it is required to use ansible_facts
+    # pylint: disable=global-variable-undefined, invalid-name,
+    # redefined-outer-name
+    global module
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            dest=dict(required=True),
+            yaml_key=dict(required=True),
+            yaml_value=dict(required=True),
+            backup=dict(required=False, default=True, type='bool'),
+        ),
+        supports_check_mode=True,
+    )
+
+    dest = module.params['dest']
+    yaml_key = module.params['yaml_key']
+    yaml_value = module.safe_eval(module.params['yaml_value'])
+    backup = module.params['backup']
+
+    # Represent null values as an empty string.
+    # pylint: disable=missing-docstring, unused-argument
+    def none_representer(dumper, data):
+        return yaml.ScalarNode(tag=u'tag:yaml.org,2002:null', value=u'')
+    yaml.add_representer(type(None), none_representer)
+
+    try:
+        changes = []
+
+        yaml_file = open(dest)
+        yaml_data = yaml.safe_load(yaml_file.read())
+        yaml_file.close()
+
+        ptr = yaml_data
+        for key in yaml_key.split('.'):
+            if key not in ptr and key != yaml_key.split('.')[-1]:
+                ptr[key] = {}
+            elif key == yaml_key.split('.')[-1]:
+                if (key in ptr and module.safe_eval(ptr[key]) != yaml_value) or (key not in ptr):
+                    ptr[key] = yaml_value
+                    changes.append((yaml_key, yaml_value))
+            else:
+                ptr = ptr[key]
+
+        if len(changes) > 0:
+            if backup:
+                module.backup_local(dest)
+            yaml_file = open(dest, 'w')
+            yaml_string = yaml.dump(yaml_data, default_flow_style=False)
+            yaml_string = yaml_string.replace('\'\'', '""')
+            yaml_file.write(yaml_string)
+            yaml_file.close()
+
+        return module.exit_json(changed=(len(changes) > 0), changes=changes)
+
+    # ignore broad-except error to avoid stack trace to ansible user
+    # pylint: disable=broad-except
+    except Exception, e:
+        return module.fail_json(msg=str(e))
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+# import module snippets
+from ansible.module_utils.basic import *
+
+if __name__ == '__main__':
+    main()
diff --git a/playbooks/common/openshift-master/scaleup.yml b/playbooks/common/openshift-master/scaleup.yml
new file mode 100644
index 000000000..6f8151d30
--- /dev/null
+++ b/playbooks/common/openshift-master/scaleup.yml
@@ -0,0 +1,55 @@
+---
+- include: ../openshift-cluster/evaluate_groups.yml
+
+- name: Gather facts
+  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
+  roles:
+  - openshift_facts
+
+- name: Update master count
+  hosts: oo_masters:!oo_masters_to_config
+  serial: 1
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}"
+        master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}"
+  - name: Update master count
+    modify_yaml:
+      dest: "{{ openshift.common.config_base}}/master/master-config.yaml"
+      yaml_key: 'kubernetesMasterConfig.masterCount'
+      yaml_value: "{{ openshift.master.master_count }}"
+    notify:
+    - restart master api
+    - restart master controllers
+  handlers:
+  - name: restart master api
+    service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+    notify: verify api server
+  - name: restart master controllers
+    service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  - name: verify api server
+    command: >
+      curl -k --head --silent {{ openshift.master.api_url }}
+    register: api_available_output
+    until: api_available_output.stdout.find("200 OK") != -1
+    retries: 120
+    delay: 1
+    changed_when: false
+
+- name: Configure docker hosts
+  hosts: oo_masters_to-config:oo_nodes_to_config
+  vars:
+    docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}"
+    docker_insecure_registries: "{{ lookup('oo_option',  'docker_insecure_registries') | oo_split }}"
+    docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}"
+  roles:
+  - openshift_facts
+  - openshift_docker
+
+- include: ../openshift-master/config.yml
+
+- include: ../openshift-node/config.yml
diff --git a/playbooks/common/openshift-cluster/scaleup.yml b/playbooks/common/openshift-node/scaleup.yml
index c4340902b..d36f7acea 100644
--- a/playbooks/common/openshift-cluster/scaleup.yml
+++ b/playbooks/common/openshift-node/scaleup.yml
@@ -1,5 +1,5 @@
 ---
-- include: evaluate_groups.yml
+- include: ../openshift-cluster/evaluate_groups.yml
 
 - name: Configure docker hosts
   hosts: oo_nodes_to_config
@@ -12,5 +12,3 @@
   - openshift_docker
 
 - include: ../openshift-node/config.yml
-  vars:
-    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/gce/openshift-cluster/cluster_hosts.yml b/playbooks/gce/openshift-cluster/cluster_hosts.yml
index 5cc300bbf..b989e15fa 100644
--- a/playbooks/gce/openshift-cluster/cluster_hosts.yml
+++ b/playbooks/gce/openshift-cluster/cluster_hosts.yml
@@ -10,8 +10,12 @@ g_nfs_hosts:     "{{ g_all_hosts | intersect(groups['tag_host-type-nfs'] | defau
 
 g_master_hosts:  "{{ g_all_hosts | intersect(groups['tag_host-type-master'] | default([])) }}"
 
+g_new_master_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type-new-master'] | default([])) }}"
+
 g_node_hosts:    "{{ g_all_hosts | intersect(groups['tag_host-type-node'] | default([])) }}"
 
-g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type-infra'] | default([])) }}"
+g_new_node_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type-new-node'] | default([])) }}"
+
+g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type-infra']) | default([]) }}"
 
 g_compute_hosts: "{{ g_node_hosts | intersect(groups['tag_sub-host-type-compute'] | default([])) }}"
diff --git a/playbooks/libvirt/openshift-cluster/cluster_hosts.yml b/playbooks/libvirt/openshift-cluster/cluster_hosts.yml
index 5cc300bbf..b989e15fa 100644
--- a/playbooks/libvirt/openshift-cluster/cluster_hosts.yml
+++ b/playbooks/libvirt/openshift-cluster/cluster_hosts.yml
@@ -10,8 +10,12 @@ g_nfs_hosts:     "{{ g_all_hosts | intersect(groups['tag_host-type-nfs'] | defau
 
 g_master_hosts:  "{{ g_all_hosts | intersect(groups['tag_host-type-master'] | default([])) }}"
 
+g_new_master_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type-new-master'] | default([])) }}"
+
 g_node_hosts:    "{{ g_all_hosts | intersect(groups['tag_host-type-node'] | default([])) }}"
 
-g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type-infra'] | default([])) }}"
+g_new_node_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type-new-node'] | default([])) }}"
+
+g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type-infra']) | default([]) }}"
 
 g_compute_hosts: "{{ g_node_hosts | intersect(groups['tag_sub-host-type-compute'] | default([])) }}"
diff --git a/playbooks/openstack/openshift-cluster/cluster_hosts.yml b/playbooks/openstack/openshift-cluster/cluster_hosts.yml
index ca87bc655..9a3361919 100644
--- a/playbooks/openstack/openshift-cluster/cluster_hosts.yml
+++ b/playbooks/openstack/openshift-cluster/cluster_hosts.yml
@@ -10,8 +10,12 @@ g_nfs_hosts:     "{{ g_all_hosts | intersect(groups['tag_host-type_nfs'] | defau
 
 g_master_hosts:  "{{ g_all_hosts | intersect(groups['tag_host-type_master'] | default([])) }}"
 
+g_new_master_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type_new_master'] | default([])) }}"
+
 g_node_hosts:    "{{ g_all_hosts | intersect(groups['tag_host-type_node'] | default([])) }}"
 
-g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type_infra'] | default([])) }}"
+g_new_node_hosts: "{{ g_all_hosts | intersect(groups['tag_host-type_new_node'] | default([])) }}"
+
+g_infra_hosts:   "{{ g_node_hosts | intersect(groups['tag_sub-host-type_infra']) | default([]) }}"
 
 g_compute_hosts: "{{ g_node_hosts | intersect(groups['tag_sub-host-type_compute'] | default([])) }}"
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 34fbe1296..0f25881f1 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -916,41 +916,79 @@ def apply_provider_facts(facts, provider_facts):
     facts['provider'] = provider_facts
     return facts
 
-
-def merge_facts(orig, new, additive_facts_to_overwrite):
+# Disabling pylint too many branches. This function needs refactored
+# but is a very core part of openshift_facts.
+# pylint: disable=too-many-branches
+def merge_facts(orig, new, additive_facts_to_overwrite, protected_facts_to_overwrite):
     """ Recursively merge facts dicts
 
         Args:
             orig (dict): existing facts
             new (dict): facts to update
-
             additive_facts_to_overwrite (list): additive facts to overwrite in jinja
                                                 '.' notation ex: ['master.named_certificates']
+            protected_facts_to_overwrite (list): protected facts to overwrite in jinja
+                                                 '.' notation ex: ['master.master_count']
 
         Returns:
             dict: the merged facts
     """
     additive_facts = ['named_certificates']
+    protected_facts = ['ha', 'master_count']
     facts = dict()
     for key, value in orig.iteritems():
+        # Key exists in both old and new facts.
         if key in new:
+            # Continue to recurse if old and new fact is a dictionary.
             if isinstance(value, dict) and isinstance(new[key], dict):
+                # Collect the subset of additive facts to overwrite if
+                # key matches. These will be passed to the subsequent
+                # merge_facts call.
                 relevant_additive_facts = []
-                # Keep additive_facts_to_overwrite if key matches
                 for item in additive_facts_to_overwrite:
                     if '.' in item and item.startswith(key + '.'):
                         relevant_additive_facts.append(item)
-                facts[key] = merge_facts(value, new[key], relevant_additive_facts)
+
+                # Collect the subset of protected facts to overwrite
+                # if key matches. These will be passed to the
+                # subsequent merge_facts call.
+                relevant_protected_facts = []
+                for item in protected_facts_to_overwrite:
+                    if '.' in item and item.startswith(key + '.'):
+                        relevant_protected_facts.append(item)
+                facts[key] = merge_facts(value, new[key], relevant_additive_facts, relevant_protected_facts)
+            # Key matches an additive fact and we are not overwriting
+            # it so we will append the new value to the existing value.
             elif key in additive_facts and key not in [x.split('.')[-1] for x in additive_facts_to_overwrite]:
-                # Fact is additive so we'll combine orig and new.
                 if isinstance(value, list) and isinstance(new[key], list):
                     new_fact = []
                     for item in copy.deepcopy(value) + copy.deepcopy(new[key]):
                         if item not in new_fact:
                             new_fact.append(item)
                     facts[key] = new_fact
+            # Key matches a protected fact and we are not overwriting
+            # it so we will determine if it is okay to change this
+            # fact.
+            elif key in protected_facts and key not in [x.split('.')[-1] for x in protected_facts_to_overwrite]:
+                # The master count (int) can only increase unless it
+                # has been passed as a protected fact to overwrite.
+                if key == 'master_count':
+                    if int(value) <= int(new[key]):
+                        facts[key] = copy.deepcopy(new[key])
+                    else:
+                        module.fail_json(msg='openshift_facts received a lower value for openshift.master.master_count')
+                # ha (bool) can not change unless it has been passed
+                # as a protected fact to overwrite.
+                if key == 'ha':
+                    if bool(value) != bool(new[key]):
+                        module.fail_json(msg='openshift_facts received a different value for openshift.master.ha')
+                    else:
+                        facts[key] = value
+            # No other condition has been met. Overwrite the old fact
+            # with the new value.
             else:
                 facts[key] = copy.deepcopy(new[key])
+        # Key isn't in new so add it to facts to keep it.
         else:
             facts[key] = copy.deepcopy(value)
     new_keys = set(new.keys()) - set(orig.keys())
@@ -1114,6 +1152,8 @@ class OpenShiftFacts(object):
             local_facts (dict): local facts to set
             additive_facts_to_overwrite (list): additive facts to overwrite in jinja
                                                 '.' notation ex: ['master.named_certificates']
+            protected_facts_to_overwrite (list): protected facts to overwrite in jinja
+                                                 '.' notation ex: ['master.master_count']
 
         Raises:
             OpenShiftFactsUnsupportedRoleError:
@@ -1122,7 +1162,10 @@ class OpenShiftFacts(object):
 
     # Disabling too-many-arguments, this should be cleaned up as a TODO item.
     # pylint: disable=too-many-arguments
-    def __init__(self, role, filename, local_facts, additive_facts_to_overwrite=False, openshift_env=None):
+    def __init__(self, role, filename, local_facts,
+                 additive_facts_to_overwrite=None,
+                 openshift_env=None,
+                 protected_facts_to_overwrite=None):
         self.changed = False
         self.filename = filename
         if role not in self.known_roles:
@@ -1131,27 +1174,41 @@ class OpenShiftFacts(object):
             )
         self.role = role
         self.system_facts = ansible_facts(module)
-        self.facts = self.generate_facts(local_facts, additive_facts_to_overwrite, openshift_env)
-
-    def generate_facts(self, local_facts, additive_facts_to_overwrite, openshift_env):
+        self.facts = self.generate_facts(local_facts,
+                                         additive_facts_to_overwrite,
+                                         openshift_env,
+                                         protected_facts_to_overwrite)
+
+    def generate_facts(self,
+                       local_facts,
+                       additive_facts_to_overwrite,
+                       openshift_env,
+                       protected_facts_to_overwrite):
         """ Generate facts
 
             Args:
-                local_facts (dict): local_facts for overriding generated
-                                    defaults
+                local_facts (dict): local_facts for overriding generated defaults
                 additive_facts_to_overwrite (list): additive facts to overwrite in jinja
                                                     '.' notation ex: ['master.named_certificates']
-
+                openshift_env (dict): openshift_env facts for overriding generated defaults
+                protected_facts_to_overwrite (list): protected facts to overwrite in jinja
+                                                     '.' notation ex: ['master.master_count']
             Returns:
                 dict: The generated facts
         """
-        local_facts = self.init_local_facts(local_facts, additive_facts_to_overwrite, openshift_env)
+        local_facts = self.init_local_facts(local_facts,
+                                            additive_facts_to_overwrite,
+                                            openshift_env,
+                                            protected_facts_to_overwrite)
         roles = local_facts.keys()
 
         defaults = self.get_defaults(roles)
         provider_facts = self.init_provider_facts()
         facts = apply_provider_facts(defaults, provider_facts)
-        facts = merge_facts(facts, local_facts, additive_facts_to_overwrite)
+        facts = merge_facts(facts,
+                            local_facts,
+                            additive_facts_to_overwrite,
+                            protected_facts_to_overwrite)
         facts['current_config'] = get_current_config(facts)
         facts = set_url_facts_if_unset(facts)
         facts = set_project_cfg_facts_if_unset(facts)
@@ -1315,13 +1372,20 @@ class OpenShiftFacts(object):
 
     # Disabling too-many-branches. This should be cleaned up as a TODO item.
     #pylint: disable=too-many-branches
-    def init_local_facts(self, facts=None, additive_facts_to_overwrite=False, openshift_env=None):
+    def init_local_facts(self, facts=None,
+                         additive_facts_to_overwrite=None,
+                         openshift_env=None,
+                         protected_facts_to_overwrite=None):
         """ Initialize the provider facts
 
             Args:
                 facts (dict): local facts to set
                 additive_facts_to_overwrite (list): additive facts to overwrite in jinja
                                                     '.' notation ex: ['master.named_certificates']
+                openshift_env (dict): openshift env facts to set
+                protected_facts_to_overwrite (list): protected facts to overwrite in jinja
+                                                     '.' notation ex: ['master.master_count']
+
 
             Returns:
                 dict: The result of merging the provided facts with existing
@@ -1347,7 +1411,10 @@ class OpenShiftFacts(object):
                     elif key not in current_level:
                         current_level[key] = dict()
                         current_level = current_level[key]
-                facts_to_set = merge_facts(facts_to_set, oo_env_facts, [])
+                facts_to_set = merge_facts(orig=facts_to_set,
+                                           new=oo_env_facts,
+                                           additive_facts_to_overwrite=[],
+                                           protected_facts_to_overwrite=[])
 
         local_facts = get_local_facts_from_file(self.filename)
 
@@ -1356,7 +1423,10 @@ class OpenShiftFacts(object):
                                                   basestring):
                 facts_to_set[arg] = module.from_json(facts_to_set[arg])
 
-        new_local_facts = merge_facts(local_facts, facts_to_set, additive_facts_to_overwrite)
+        new_local_facts = merge_facts(local_facts,
+                                      facts_to_set,
+                                      additive_facts_to_overwrite,
+                                      protected_facts_to_overwrite)
         for facts in new_local_facts.values():
             keys_to_delete = []
             if isinstance(facts, dict):
@@ -1452,7 +1522,8 @@ def main():
                       choices=OpenShiftFacts.known_roles),
             local_facts=dict(default=None, type='dict', required=False),
             additive_facts_to_overwrite=dict(default=[], type='list', required=False),
-            openshift_env=dict(default={}, type='dict', required=False)
+            openshift_env=dict(default={}, type='dict', required=False),
+            protected_facts_to_overwrite=dict(default=[], type='list', required=False),
         ),
         supports_check_mode=True,
         add_file_common_args=True,
@@ -1462,6 +1533,7 @@ def main():
     local_facts = module.params['local_facts']
     additive_facts_to_overwrite = module.params['additive_facts_to_overwrite']
     openshift_env = module.params['openshift_env']
+    protected_facts_to_overwrite = module.params['protected_facts_to_overwrite']
 
     fact_file = '/etc/ansible/facts.d/openshift.fact'
 
@@ -1469,7 +1541,8 @@ def main():
                                      fact_file,
                                      local_facts,
                                      additive_facts_to_overwrite,
-                                     openshift_env)
+                                     openshift_env,
+                                     protected_facts_to_overwrite)
 
     file_params = module.params.copy()
     file_params['path'] = fact_file
diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml
index 6d9be81c0..66960e73e 100644
--- a/roles/openshift_master_ca/tasks/main.yml
+++ b/roles/openshift_master_ca/tasks/main.yml
@@ -25,4 +25,4 @@
       --master={{ openshift.master.api_url }}
       --public-master={{ openshift.master.public_api_url }}
       --cert-dir={{ openshift_master_config_dir }} --overwrite=false
-  when: master_certs_missing
+  when: master_certs_missing | bool
diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml
index 7c58e943a..72869a592 100644
--- a/roles/openshift_master_certificates/tasks/main.yml
+++ b/roles/openshift_master_certificates/tasks/main.yml
@@ -6,40 +6,16 @@
     mode: 0700
   with_items: masters_needing_certs
 
-- set_fact:
-    master_certificates:
-    - ca.crt
-    - ca.key
-    - ca.serial.txt
-    - admin.crt
-    - admin.key
-    - admin.kubeconfig
-    - master.kubelet-client.crt
-    - master.kubelet-client.key
-    - master.server.crt
-    - master.server.key
-    - openshift-master.crt
-    - openshift-master.key
-    - openshift-master.kubeconfig
-    - openshift-registry.crt
-    - openshift-registry.key
-    - openshift-registry.kubeconfig
-    - openshift-router.crt
-    - openshift-router.key
-    - openshift-router.kubeconfig
-    - serviceaccounts.private.key
-    - serviceaccounts.public.key
-    master_31_certificates:
-    - master.proxy-client.crt
-    - master.proxy-client.key
-
 - file:
     src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
     dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
     state: hard
   with_nested:
   - masters_needing_certs
-  - "{{ master_certificates | union(master_31_certificates) if openshift.common.version_gte_3_1_or_1_1 | bool else master_certificates }}"
+  -
+    - ca.crt
+    - ca.key
+    - ca.serial.txt
 
 - name: Create the master certificates if they do not already exist
   command: >
@@ -49,5 +25,5 @@
       --public-master={{ item.openshift.master.public_api_url }}
       --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
       --overwrite=false
-  when: master_certs_missing
+  when: item.master_certs_missing | bool
   with_items: masters_needing_certs
diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py
index 64875dbe9..f2c7289fa 100644
--- a/utils/src/ooinstall/openshift_ansible.py
+++ b/utils/src/ooinstall/openshift_ansible.py
@@ -206,7 +206,7 @@ def run_main_playbook(hosts, hosts_to_run_on, verbose=False):
     inventory_file = generate_inventory(hosts_to_run_on)
     if len(hosts_to_run_on) != len(hosts):
         main_playbook_path = os.path.join(CFG.ansible_playbook_directory,
-                                          'playbooks/byo/openshift-cluster/scaleup.yml')
+                                          'playbooks/byo/openshift-node/scaleup.yml')
     else:
         main_playbook_path = os.path.join(CFG.ansible_playbook_directory,
                                           'playbooks/byo/openshift-cluster/config.yml')