Merge pull request #6145 from mtnbikenc/consolidate-openshift-etcd

Playbook Consolidation - openshift-etcd

18 files changed, 568 insertions, 0 deletions

diff --git a/playbooks/openshift-etcd/certificates.yml b/playbooks/openshift-etcd/certificates.yml
new file mode 100644
index 000000000..8240d3fe6
--- /dev/null
+++ b/playbooks/openshift-etcd/certificates.yml
@@ -0,0 +1,6 @@
+---
+- include: ../init/main.yml
+
+- include: private/ca.yml
+
+- include: private/certificates.yml
diff --git a/playbooks/openshift-etcd/config.yml b/playbooks/openshift-etcd/config.yml
new file mode 100644
index 000000000..8ee57ce8d
--- /dev/null
+++ b/playbooks/openshift-etcd/config.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/config.yml
diff --git a/playbooks/openshift-etcd/embedded2external.yml b/playbooks/openshift-etcd/embedded2external.yml
new file mode 100644
index 000000000..a11b140de
--- /dev/null
+++ b/playbooks/openshift-etcd/embedded2external.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/embedded2external.yml
diff --git a/playbooks/openshift-etcd/migrate.yml b/playbooks/openshift-etcd/migrate.yml
new file mode 100644
index 000000000..a307c2740
--- /dev/null
+++ b/playbooks/openshift-etcd/migrate.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/migrate.yml
diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml
new file mode 100644
index 000000000..ac5543be9
--- /dev/null
+++ b/playbooks/openshift-etcd/private/ca.yml
@@ -0,0 +1,15 @@
+---
+- name: Generate new etcd CA
+  hosts: oo_first_etcd
+  roles:
+  - role: openshift_etcd_facts
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: ca
+    vars:
+      etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    when:
+    - etcd_ca_setup | default(True) | bool
diff --git a/playbooks/openshift-etcd/private/certificates.yml b/playbooks/openshift-etcd/private/certificates.yml
new file mode 100644
index 000000000..eb6b94f33
--- /dev/null
+++ b/playbooks/openshift-etcd/private/certificates.yml
@@ -0,0 +1,4 @@
+---
+- include: server_certificates.yml
+
+- include: master_etcd_certificates.yml
diff --git a/playbooks/openshift-etcd/private/config.yml b/playbooks/openshift-etcd/private/config.yml
new file mode 100644
index 000000000..3fe483785
--- /dev/null
+++ b/playbooks/openshift-etcd/private/config.yml
@@ -0,0 +1,40 @@
+---
+- name: etcd Install Checkpoint Start
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set etcd install 'In Progress'
+    run_once: true
+    set_stats:
+      data:
+        installer_phase_etcd:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
+
+- include: ca.yml
+
+- include: certificates.yml
+
+- name: Configure etcd
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+  - role: os_firewall
+  - role: openshift_etcd
+    etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  - role: nickhammond.logrotate
+
+- name: etcd Install Checkpoint End
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Set etcd install 'Complete'
+    run_once: true
+    set_stats:
+      data:
+        installer_phase_etcd:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml
new file mode 100644
index 000000000..0bd9912b0
--- /dev/null
+++ b/playbooks/openshift-etcd/private/embedded2external.yml
@@ -0,0 +1,172 @@
+---
+- name: Pre-migrate checks
+  hosts: localhost
+  tasks:
+  # Check there is only one etcd host
+  - assert:
+      that: groups.oo_etcd_to_config | default([]) | length == 1
+      msg: "[etcd] group must contain only one host"
+  # Check there is only one master
+  - assert:
+      that: groups.oo_masters_to_config | default([]) | length == 1
+      msg: "[master] group must contain only one host"
+
+# 1. stop a master
+- name: Prepare masters for etcd data migration
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  tasks:
+  - name: Check the master API is ready
+    include_role:
+      name: openshift_master
+      tasks_from: check_master_api_is_ready
+  - set_fact:
+      master_service: "{{ openshift.common.service_type + '-master' }}"
+      embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+  - debug:
+      msg: "master service name: {{ master_service }}"
+  - name: Stop master
+    service:
+      name: "{{ master_service }}"
+      state: stopped
+  # 2. backup embedded etcd
+  # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285
+  - include_role:
+      name: etcd
+      tasks_from: backup
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.archive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+# 3. deploy certificates (for etcd and master)
+- include: ca.yml
+
+- include: server_certificates.yml
+
+- name: Backup etcd client certificates for master host
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup_master_etcd_certificates
+
+- name: Redeploy master etcd certificates
+  include: master_etcd_certificates.yml
+  vars:
+    etcd_certificates_redeploy: "{{ true }}"
+
+# 4. deploy external etcd
+- include: config.yml
+
+# 5. stop external etcd
+- name: Cleanse etcd
+  hosts: oo_etcd_to_config[0]
+  gather_facts: no
+  pre_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: disable_etcd
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  - include_role:
+      name: etcd
+      tasks_from: clean_data
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+
+# 6. copy the embedded etcd backup to the external host
+# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory
+- name: Copy embedded etcd backup to the external host
+  hosts: localhost
+  tasks:
+  - name: Create local temp directory for syncing etcd backup
+    local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX
+    register: g_etcd_client_mktemp
+    changed_when: False
+    become: no
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.fetch
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_first_master.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_first_master[0] }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.copy
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_etcd_to_config[0] }}"
+
+  - debug:
+      msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}"
+
+  - name: Delete temporary directory
+    local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent
+    changed_when: False
+    become: no
+
+# 7. force new cluster from the backup
+- name: Force new etcd cluster
+  hosts: oo_etcd_to_config[0]
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup.unarchive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.force_new_cluster
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+
+# 8. re-configure master to use the external etcd
+- name: Configure master to use external etcd
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: openshift_master
+      tasks_from: configure_external_etcd
+    vars:
+      etcd_peer_url_scheme: "https"
+      etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}"
+      etcd_peer_port: 2379
+
+  # 9. start the master
+  - name: Start master
+    service:
+      name: "{{ master_service }}"
+      state: started
+    register: service_status
+    until: service_status.state is defined and service_status.state == "started"
+    retries: 5
+    delay: 10
diff --git a/playbooks/openshift-etcd/private/filter_plugins b/playbooks/openshift-etcd/private/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/openshift-etcd/private/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/openshift-etcd/private/lookup_plugins b/playbooks/openshift-etcd/private/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/openshift-etcd/private/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/openshift-etcd/private/master_etcd_certificates.yml b/playbooks/openshift-etcd/private/master_etcd_certificates.yml
new file mode 100644
index 000000000..0a25aac57
--- /dev/null
+++ b/playbooks/openshift-etcd/private/master_etcd_certificates.yml
@@ -0,0 +1,14 @@
+---
+- name: Create etcd client certificates for master hosts
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+    - role: openshift_etcd_client_certificates
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: "master.etcd-"
+      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml
new file mode 100644
index 000000000..31362f2f6
--- /dev/null
+++ b/playbooks/openshift-etcd/private/migrate.yml
@@ -0,0 +1,169 @@
+---
+- name: Check if the master has embedded etcd
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tags:
+  - always
+  tasks:
+  - fail:
+      msg: "Migration of an embedded etcd is not supported. Please, migrate the embedded etcd into an external etcd first."
+    when:
+    - groups.oo_etcd_to_config | default([]) | length == 0
+
+- name: Run pre-checks
+  hosts: oo_etcd_to_migrate
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: migrate.pre_check
+    vars:
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      etcd_peer: "{{ ansible_default_ipv4.address }}"
+
+# TODO: This will be different for release-3.6 branch
+- name: Prepare masters for etcd data migration
+  hosts: oo_masters_to_config
+  tasks:
+  - set_fact:
+      master_services:
+      - "{{ openshift.common.service_type + '-master-controllers' }}"
+      - "{{ openshift.common.service_type + '-master-api' }}"
+  - debug:
+      msg: "master service name: {{ master_services }}"
+  - name: Stop masters
+    service:
+      name: "{{ item }}"
+      state: stopped
+    with_items: "{{ master_services }}"
+
+- name: Backup v2 data
+  hosts: oo_etcd_to_migrate
+  gather_facts: no
+  roles:
+  - role: openshift_facts
+  post_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migration
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+
+- name: Gate on etcd backup
+  hosts: localhost
+  connection: local
+  become: no
+  tasks:
+  - set_fact:
+      etcd_backup_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_etcd_to_migrate)
+                                 | oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}"
+  - set_fact:
+      etcd_backup_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_backup_completed) | list }}"
+  - fail:
+      msg: "Migration cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
+    when:
+    - etcd_backup_failed | length > 0
+
+- name: Stop etcd
+  hosts: oo_etcd_to_migrate
+  gather_facts: no
+  pre_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: disable_etcd
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+
+- name: Migrate data on first etcd
+  hosts: oo_etcd_to_migrate[0]
+  gather_facts: no
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: migrate
+    vars:
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+
+- name: Clean data stores on remaining etcd hosts
+  hosts: oo_etcd_to_migrate[1:]
+  gather_facts: no
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: clean_data
+    vars:
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+  - name: Add etcd hosts
+    delegate_to: localhost
+    add_host:
+      name: "{{ item }}"
+      groups: oo_new_etcd_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_become: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ groups.oo_etcd_to_migrate[1:] | default([]) }}"
+    changed_when: no
+  - name: Set success
+    set_fact:
+      r_etcd_migrate_success: true
+
+- include: ./scaleup.yml
+
+- name: Gate on etcd migration
+  hosts: oo_masters_to_config
+  gather_facts: no
+  tasks:
+  - set_fact:
+      etcd_migration_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_etcd_to_migrate)
+                                 | oo_collect('inventory_hostname', {'r_etcd_migrate_success': true}) }}"
+  - set_fact:
+      etcd_migration_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_migration_completed) | list }}"
+
+- name: Add TTLs on the first master
+  hosts: oo_first_master[0]
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: migrate.add_ttls
+    vars:
+      etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+    when: etcd_migration_failed | length == 0
+
+- name: Configure masters if etcd data migration is succesfull
+  hosts: oo_masters_to_config
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: migrate.configure_master
+    when: etcd_migration_failed | length == 0
+  - debug:
+      msg: "Skipping master re-configuration since migration failed."
+    when:
+    - etcd_migration_failed | length > 0
+  - name: Start master services
+    service:
+      name: "{{ item }}"
+      state: started
+    register: service_status
+    # Sometimes the master-api, resp. master-controllers fails to start for the first time
+    until: service_status.state is defined and service_status.state == "started"
+    retries: 5
+    delay: 10
+    with_items: "{{ master_services[::-1] }}"
+  - fail:
+      msg: "Migration failed. The following hosts were not properly migrated: {{ etcd_migration_failed | join(',') }}"
+    when:
+    - etcd_migration_failed | length > 0
diff --git a/playbooks/openshift-etcd/private/restart.yml b/playbooks/openshift-etcd/private/restart.yml
new file mode 100644
index 000000000..5eaea5ae8
--- /dev/null
+++ b/playbooks/openshift-etcd/private/restart.yml
@@ -0,0 +1,27 @@
+---
+- name: Restart etcd
+  hosts: oo_etcd_to_config
+  serial: 1
+  tasks:
+    - name: restart etcd
+      service:
+        name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}"
+        state: restarted
+      when:
+        - not g_etcd_certificates_expired | default(false) | bool
+
+- name: Restart etcd
+  hosts: oo_etcd_to_config
+  tasks:
+    - name: stop etcd
+      service:
+        name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}"
+        state: stopped
+      when:
+        - g_etcd_certificates_expired | default(false) | bool
+    - name: start etcd
+      service:
+        name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}"
+        state: started
+      when:
+        - g_etcd_certificates_expired | default(false) | bool
diff --git a/playbooks/openshift-etcd/private/roles b/playbooks/openshift-etcd/private/roles
new file mode 120000
index 000000000..20c4c58cf
--- /dev/null
+++ b/playbooks/openshift-etcd/private/roles
@@ -0,0 +1 @@
+../../../roles
\ No newline at end of file
diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml
new file mode 100644
index 000000000..20061366c
--- /dev/null
+++ b/playbooks/openshift-etcd/private/scaleup.yml
@@ -0,0 +1,83 @@
+---
+- name: Gather facts
+  hosts: oo_etcd_to_config:oo_new_etcd_to_config
+  roles:
+  - openshift_etcd_facts
+  post_tasks:
+  - set_fact:
+      etcd_hostname: "{{ etcd_hostname }}"
+      etcd_ip: "{{ etcd_ip }}"
+
+- name: Configure etcd
+  hosts: oo_new_etcd_to_config
+  serial: 1
+  any_errors_fatal: true
+  vars:
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+  pre_tasks:
+  - name: Add new etcd members to cluster
+    command: >
+      /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }}
+                       --key-file {{ etcd_peer_key_file }}
+                       --ca-file {{ etcd_peer_ca_file }}
+                       -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }}
+                       member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }}
+    delegate_to: "{{ etcd_ca_host }}"
+    failed_when:
+    - etcd_add_check.rc == 1
+    - ("peerURL exists" not in etcd_add_check.stderr)
+    register: etcd_add_check
+    retries: 3
+    delay: 10
+    until: etcd_add_check.rc == 0
+  - include_role:
+      name: etcd
+      tasks_from: server_certificates
+    vars:
+      etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
+      etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  roles:
+  - role: os_firewall
+    when: etcd_add_check.rc == 0
+  - role: openshift_etcd
+    when: etcd_add_check.rc == 0
+    etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}"
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    etcd_initial_cluster_state: "existing"
+    etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}"
+    etcd_ca_setup: False
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  - role: nickhammond.logrotate
+    when: etcd_add_check.rc == 0
+  post_tasks:
+  - name: Verify cluster is stable
+    command: >
+      /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }}
+                       --key-file {{ etcd_peer_key_file }}
+                       --ca-file {{ etcd_peer_ca_file }}
+                       -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }}
+                       cluster-health
+    register: scaleup_health
+    retries: 3
+    delay: 30
+    until: scaleup_health.rc == 0
+
+- name: Update master etcd client urls
+  hosts: oo_masters_to_config
+  serial: 1
+  vars:
+    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_master_etcd_hosts: "{{ hostvars
+                                     | oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) ))
+                                     | oo_collect('openshift.common.hostname')
+                                     | default(none, true) }}"
+    openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}"
+  roles:
+  - role: openshift_master_facts
+  post_tasks:
+  - include_role:
+      name: openshift_master
+      tasks_from: update_etcd_client_urls
diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml
new file mode 100644
index 000000000..10e06747b
--- /dev/null
+++ b/playbooks/openshift-etcd/private/server_certificates.yml
@@ -0,0 +1,15 @@
+---
+- name: Create etcd server certificates for etcd hosts
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+  post_tasks:
+    - include_role:
+        name: etcd
+        tasks_from: server_certificates
+      vars:
+        etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
diff --git a/playbooks/openshift-etcd/restart.yml b/playbooks/openshift-etcd/restart.yml
new file mode 100644
index 000000000..5e28e274e
--- /dev/null
+++ b/playbooks/openshift-etcd/restart.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/restart.yml
diff --git a/playbooks/openshift-etcd/scaleup.yml b/playbooks/openshift-etcd/scaleup.yml
new file mode 100644
index 000000000..d83697131
--- /dev/null
+++ b/playbooks/openshift-etcd/scaleup.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/scaleup.yml