Merge pull request #1337 from abutcher/master-scaleup

Support for adding new masters

7 files changed, 234 insertions, 75 deletions

diff --git a/playbooks/common/openshift-cluster/additional_config.yml b/playbooks/common/openshift-cluster/additional_config.yml
new file mode 100644
index 000000000..0f4e6ab88
--- /dev/null
+++ b/playbooks/common/openshift-cluster/additional_config.yml
@@ -0,0 +1,55 @@
+- name: Configure flannel
+  hosts: oo_first_master
+  vars:
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+  roles:
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+
+- name: Additional master configuration
+  hosts: oo_first_master
+  vars:
+    cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+    openshift_master_ha: "{{ groups.oo_masters | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters | join(' ')}}"
+  roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker"
+  - role: openshift_examples
+    when: openshift.common.install_examples | bool
+  - role: openshift_cluster_metrics
+    when: openshift.common.use_cluster_metrics | bool
+  - role: openshift_manageiq
+    when: openshift.common.use_manageiq | bool
+  - role: cockpit
+    when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
+      (osm_use_cockpit | bool or osm_use_cockpit is undefined )
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+  - role: pods
+    when: openshift.common.deployment_type == 'online'
+  - role: os_env_extras
+    when: openshift.common.deployment_type == 'online'
+
+- name: Create persistent volumes and create hosted services
+  hosts: oo_first_master
+  vars:
+    attach_registry_volume: "{{ openshift.hosted.registry.storage.kind != None }}"
+    deploy_infra: "{{ openshift.master.infra_nodes | default([]) | length > 0 }}"
+    persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups) }}"
+    persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims }}"
+  roles:
+  - role: openshift_persistent_volumes
+    when: persistent_volumes | length > 0 or persistent_volume_claims | length > 0
+  - role: openshift_serviceaccounts
+    openshift_serviceaccounts_names:
+    - router
+    - registry
+    openshift_serviceaccounts_namespace: default
+    openshift_serviceaccounts_sccs:
+    - privileged
+  - role: openshift_router
+    when: deploy_infra | bool
+  - role: openshift_registry
+    when: deploy_infra | bool and attach_registry_volume | bool
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index 2cad4b362..23c8f039e 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -11,4 +11,6 @@
 
 - include: ../openshift-master/config.yml
 
+- include: additional_config.yml
+
 - include: ../openshift-node/config.yml
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index 7917bfba5..432a92b49 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -10,8 +10,8 @@
     when: g_etcd_hosts is not defined
 
   - fail:
-      msg: This playbook requires g_master_hosts to be set
-    when: g_master_hosts is not defined
+      msg: This playbook requires g_master_hosts or g_new_master_hosts to be set
+    when: g_master_hosts is not defined and g_new_master_hosts is not defined
 
   - fail:
       msg: This playbook requires g_node_hosts or g_new_node_hosts to be set
@@ -29,6 +29,14 @@
       msg: The nfs group must be limited to one host
     when: (groups[g_nfs_hosts] | default([])) | length > 1
 
+  - name: Evaluate oo_masters
+    add_host:
+      name: "{{ item }}"
+      groups: oo_masters
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_master_hosts | union(g_new_master_hosts) | default([]) }}"
+
   - name: Evaluate oo_etcd_to_config
     add_host:
       name: "{{ item }}"
@@ -43,11 +51,7 @@
       groups: oo_masters_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_master_hosts | default([]) }}"
-
-  # Use g_new_node_hosts if it exists otherwise g_node_hosts
-  - set_fact:
-      g_node_hosts_to_config: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
+    with_items: "{{ g_new_master_hosts | default(g_master_hosts | default([], true), true) }}"
 
   - name: Evaluate oo_nodes_to_config
     add_host:
@@ -55,7 +59,7 @@
       groups: oo_nodes_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_node_hosts_to_config | default([]) }}"
+    with_items: "{{ g_new_node_hosts | default(g_node_hosts | default([], true), true) }}"
 
   # Skip adding the master to oo_nodes_to_config when g_new_node_hosts is
   - name: Evaluate oo_nodes_to_config
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 77edbd1a6..b9d595576 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -57,6 +57,8 @@
           console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
           public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
           portal_net: "{{ openshift_master_portal_net | default(None) }}"
+          ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}"
+          master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}"
   - openshift_facts:
       role: hosted
       openshift_env:
@@ -144,7 +146,7 @@
     when: etcd_client_certs_missing is defined and etcd_client_certs_missing
 
 - name: Determine if master certificates need to be generated
-  hosts: oo_masters_to_config
+  hosts: oo_first_master:oo_masters_to_config
   tasks:
   - set_fact:
       openshift_master_certs_no_etcd:
@@ -221,15 +223,6 @@
       validate_checksum: yes
     with_items: masters_needing_certs
 
-- name: Compute haproxy_backend_servers
-  hosts: localhost
-  connection: local
-  become: no
-  gather_facts: no
-  tasks:
-  - set_fact:
-      haproxy_backend_servers: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_haproxy_backend_masters }}"
-
 - name: Configure load balancers
   hosts: oo_lb_to_config
   vars:
@@ -248,11 +241,11 @@
       mode: tcp
       option: tcplog
       balance: source
-      servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
+      servers: "{{ hostvars | oo_select_keys(groups['oo_masters']) | oo_haproxy_backend_masters }}"
   roles:
   - role: openshift_facts
   - role: haproxy
-    when: groups.oo_masters_to_config | length > 1
+    when: hostvars[groups.oo_first_master.0].openshift.master.ha | bool
 
 - name: Check for cached session secrets
   hosts: oo_first_master
@@ -334,8 +327,8 @@
   serial: 1
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
+    openshift_master_ha: "{{ openshift.master.ha }}"
+    openshift_master_count: "{{ openshift.master.master_count }}"
     openshift_master_session_auth_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_auth_secrets }}"
     openshift_master_session_encryption_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_encryption_secrets }}"
   pre_tasks:
@@ -343,12 +336,12 @@
     file:
       path: "{{ openshift.common.config_base }}/master"
       state: directory
-    when: master_certs_missing and 'oo_first_master' not in group_names
+    when: master_certs_missing | bool and 'oo_first_master' not in group_names
   - name: Unarchive the tarball on the master
     unarchive:
       src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
       dest: "{{ master_cert_config_dir }}"
-    when: master_certs_missing and 'oo_first_master' not in group_names
+    when: master_certs_missing | bool and 'oo_first_master' not in group_names
   roles:
   - openshift_master
   - role: nickhammond.logrotate
@@ -359,32 +352,12 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Additional master configuration
-  hosts: oo_first_master
-  vars:
-    cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
-    etcd_urls: "{{ openshift.master.etcd_urls }}"
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
+# Additional instance config for online deployments
+- name: Additional instance config
+  hosts: oo_masters_deployment_type_online
   roles:
-  - role: openshift_master_cluster
-    when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker"
-  - role: openshift_examples
-    when: openshift.common.install_examples | bool
-  - role: openshift_cluster_metrics
-    when: openshift.common.use_cluster_metrics | bool
-  - role: openshift_manageiq
-    when: openshift.common.use_manageiq | bool
-  - role: cockpit
-    when: not openshift.common.is_atomic and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
-      (osm_use_cockpit | bool or osm_use_cockpit is undefined )
-  - role: flannel_register
-    when: openshift.common.use_flannel | bool
-  - role: pods
-    when: openshift.common.deployment_type == 'online'
-  - role: os_env_extras
-    when: openshift.common.deployment_type == 'online'
-
+  - pods
+  - os_env_extras
 
 - name: Delete temporary directory on localhost
   hosts: localhost
@@ -394,26 +367,3 @@
   tasks:
   - file: name={{ g_master_mktemp.stdout }} state=absent
     changed_when: False
-
-- name: Create persistent volumes and create hosted services
-  hosts: oo_first_master
-  vars:
-    attach_registry_volume: "{{ openshift.hosted.registry.storage.kind != None }}"
-    deploy_infra: "{{ openshift.master.infra_nodes | default([]) | length > 0 }}"
-    persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups) }}"
-    persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims }}"
-  roles:
-  - role: openshift_persistent_volumes
-    when: persistent_volumes | length > 0 or persistent_volume_claims | length > 0
-  - role: openshift_serviceaccounts
-    openshift_serviceaccounts_names:
-    - router
-    - registry
-    openshift_serviceaccounts_namespace: default
-    openshift_serviceaccounts_sccs:
-    - privileged
-  - role: openshift_router
-    when: deploy_infra | bool
-  - role: openshift_registry
-    when: deploy_infra | bool and attach_registry_volume | bool
-
diff --git a/playbooks/common/openshift-master/library/modify_yaml.py b/playbooks/common/openshift-master/library/modify_yaml.py
new file mode 100755
index 000000000..a4be10ca3
--- /dev/null
+++ b/playbooks/common/openshift-master/library/modify_yaml.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vim: expandtab:tabstop=4:shiftwidth=4
+
+''' modify_yaml ansible module '''
+
+import yaml
+
+DOCUMENTATION = '''
+---
+module: modify_yaml
+short_description: Modify yaml key value pairs
+author: Andrew Butcher
+requirements: [ ]
+'''
+EXAMPLES = '''
+- modify_yaml:
+    dest: /etc/origin/master/master-config.yaml
+    yaml_key: 'kubernetesMasterConfig.masterCount'
+    yaml_value: 2
+'''
+
+def main():
+    ''' Modify key (supplied in jinja2 dot notation) in yaml file, setting
+        the key to the desired value.
+    '''
+
+    # disabling pylint errors for global-variable-undefined and invalid-name
+    # for 'global module' usage, since it is required to use ansible_facts
+    # pylint: disable=global-variable-undefined, invalid-name,
+    # redefined-outer-name
+    global module
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            dest=dict(required=True),
+            yaml_key=dict(required=True),
+            yaml_value=dict(required=True),
+            backup=dict(required=False, default=True, type='bool'),
+        ),
+        supports_check_mode=True,
+    )
+
+    dest = module.params['dest']
+    yaml_key = module.params['yaml_key']
+    yaml_value = module.safe_eval(module.params['yaml_value'])
+    backup = module.params['backup']
+
+    # Represent null values as an empty string.
+    # pylint: disable=missing-docstring, unused-argument
+    def none_representer(dumper, data):
+        return yaml.ScalarNode(tag=u'tag:yaml.org,2002:null', value=u'')
+    yaml.add_representer(type(None), none_representer)
+
+    try:
+        changes = []
+
+        yaml_file = open(dest)
+        yaml_data = yaml.safe_load(yaml_file.read())
+        yaml_file.close()
+
+        ptr = yaml_data
+        for key in yaml_key.split('.'):
+            if key not in ptr and key != yaml_key.split('.')[-1]:
+                ptr[key] = {}
+            elif key == yaml_key.split('.')[-1]:
+                if (key in ptr and module.safe_eval(ptr[key]) != yaml_value) or (key not in ptr):
+                    ptr[key] = yaml_value
+                    changes.append((yaml_key, yaml_value))
+            else:
+                ptr = ptr[key]
+
+        if len(changes) > 0:
+            if backup:
+                module.backup_local(dest)
+            yaml_file = open(dest, 'w')
+            yaml_string = yaml.dump(yaml_data, default_flow_style=False)
+            yaml_string = yaml_string.replace('\'\'', '""')
+            yaml_file.write(yaml_string)
+            yaml_file.close()
+
+        return module.exit_json(changed=(len(changes) > 0), changes=changes)
+
+    # ignore broad-except error to avoid stack trace to ansible user
+    # pylint: disable=broad-except
+    except Exception, e:
+        return module.fail_json(msg=str(e))
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+# import module snippets
+from ansible.module_utils.basic import *
+
+if __name__ == '__main__':
+    main()
diff --git a/playbooks/common/openshift-master/scaleup.yml b/playbooks/common/openshift-master/scaleup.yml
new file mode 100644
index 000000000..6f8151d30
--- /dev/null
+++ b/playbooks/common/openshift-master/scaleup.yml
@@ -0,0 +1,55 @@
+---
+- include: ../openshift-cluster/evaluate_groups.yml
+
+- name: Gather facts
+  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
+  roles:
+  - openshift_facts
+
+- name: Update master count
+  hosts: oo_masters:!oo_masters_to_config
+  serial: 1
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        ha: "{{ openshift_master_ha | default(groups.oo_masters | length > 1) }}"
+        master_count: "{{ openshift_master_count | default(groups.oo_masters | length) }}"
+  - name: Update master count
+    modify_yaml:
+      dest: "{{ openshift.common.config_base}}/master/master-config.yaml"
+      yaml_key: 'kubernetesMasterConfig.masterCount'
+      yaml_value: "{{ openshift.master.master_count }}"
+    notify:
+    - restart master api
+    - restart master controllers
+  handlers:
+  - name: restart master api
+    service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+    notify: verify api server
+  - name: restart master controllers
+    service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  - name: verify api server
+    command: >
+      curl -k --head --silent {{ openshift.master.api_url }}
+    register: api_available_output
+    until: api_available_output.stdout.find("200 OK") != -1
+    retries: 120
+    delay: 1
+    changed_when: false
+
+- name: Configure docker hosts
+  hosts: oo_masters_to-config:oo_nodes_to_config
+  vars:
+    docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}"
+    docker_insecure_registries: "{{ lookup('oo_option',  'docker_insecure_registries') | oo_split }}"
+    docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}"
+  roles:
+  - openshift_facts
+  - openshift_docker
+
+- include: ../openshift-master/config.yml
+
+- include: ../openshift-node/config.yml
diff --git a/playbooks/common/openshift-cluster/scaleup.yml b/playbooks/common/openshift-node/scaleup.yml
index c4340902b..d36f7acea 100644
--- a/playbooks/common/openshift-cluster/scaleup.yml
+++ b/playbooks/common/openshift-node/scaleup.yml
@@ -1,5 +1,5 @@
 ---
-- include: evaluate_groups.yml
+- include: ../openshift-cluster/evaluate_groups.yml
 
 - name: Configure docker hosts
   hosts: oo_nodes_to_config
@@ -12,5 +12,3 @@
   - openshift_docker
 
 - include: ../openshift-node/config.yml
-  vars:
-    openshift_deployment_type: "{{ deployment_type }}"