Merge pull request #2963 from sdodson/upgrade_etcd_by_default

Upgrade etcd by default

4 files changed, 118 insertions, 104 deletions

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index cfb273b82..691961382 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -41,11 +41,15 @@
         {{ avail_disk.stdout }} Kb available.
     when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
 
-  # for non containerized etcd is already installed, don't touch it, but for containerized
-  # but not atomic always get the latest
-  - name: Install latest text for containerized but not atomic
+  # TODO - Refactor containerized backup to use etcd_container to backup the data so we don't rely on
+  # the host's etcdctl binary which may be of a different version.
+
+  # for non containerized and non embedded we should have the correct version of etcd installed already
+  # For embedded we need to use the latest because OCP 3.3 uses a version of etcd that can only be backed
+  # up with etcd-3.x
+  - name: Install latest etcd for containerized or embedded
     package: name=etcd state=latest
-    when: not openshift.common.is_atomic | bool and openshift.common.is_containerized
+    when: ( openshift.common.is_containerized and not openshift.common.is_atomic ) or embedded_etcd | bool
 
   - name: Generate etcd backup
     command: >
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
index c67cf282f..8268adc2e 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
@@ -9,119 +9,36 @@
   tags:
   - always
 
-- name: Evaluate additional groups for upgrade
+# We use two groups one for hosts we're upgrading which doesn't include embedded etcd
+# The other for backing up which includes the embedded etcd host, there's no need to
+# upgrade embedded etcd that just happens when the master is updated.
+- name: Evaluate additional groups for etcd
   hosts: localhost
   connection: local
   become: no
   tasks:
-  - fail:
-      msg: 'The etcd upgrade playbook does not support upgrading embedded etcd, simply run the normal playbooks and etcd will be upgraded when your master is updated.'
-    when:  "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
   - name: Evaluate etcd_hosts_to_upgrade
     add_host:
       name: "{{ item }}"
-      groups: etcd_hosts_to_upgrade, etcd_hosts_to_backup
+      groups: etcd_hosts_to_upgrade
+    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else [] }}"
+  - name: Evaluate etcd_hosts_to_backup
+    add_host:
+      name: "{{ item }}"
+      groups: etcd_hosts_to_backup
     with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}"
 
 - name: Backup etcd before upgrading anything
   include: backup.yml
   vars:
     backup_tag: "pre-upgrade-"
+  when: openshift_etcd_backup | default(true) | bool
 
 - name: Drop etcdctl profiles
   hosts: etcd_hosts_to_upgrade
   tasks:
   - include: roles/etcd/tasks/etcdctl.yml
 
-- name: Determine etcd version
-  hosts: etcd_hosts_to_upgrade
-  tasks:
-  - name: Record RPM based etcd version
-    command: rpm -qa --qf '%{version}' etcd\*
-    args:
-      warn: no
-    register: etcd_rpm_version
-    failed_when: false
-    when: not openshift.common.is_containerized | bool
-  - name: Record containerized etcd version
-    command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\*
-    register: etcd_container_version
-    failed_when: false
-    when: openshift.common.is_containerized | bool
-
-# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
-# through hosts, then loop through tasks only when appropriate
-- name: Upgrade to 2.1
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: '2.1'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
-
-- name: Upgrade RPM hosts to 2.2
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: '2.2'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
-
-- name: Upgrade containerized hosts to 2.2.5
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: 2.2.5
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool
-
-- name: Upgrade RPM hosts to 2.3
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: '2.3'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
-
-- name: Upgrade containerized hosts to 2.3.7
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: 2.3.7
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool
-
-- name: Upgrade RPM hosts to 3.0
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: '3.0'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
-
-- name: Upgrade containerized hosts to etcd3 image
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  vars:
-    upgrade_version: 3.0.14
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool
-
-- name: Upgrade fedora to latest
-  hosts: etcd_hosts_to_upgrade
-  serial: 1
-  tasks:
-  - include: fedora_tasks.yml
-    when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool
-
-- name: Backup etcd
-  include: backup.yml
-  vars:
-    backup_tag: "post-3.0-"
+- name: Perform etcd upgrade
+  include: ./upgrade.yml
+  when: openshift_etcd_upgrade | default(true) | bool
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml
new file mode 100644
index 000000000..5ff9521ec
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml
@@ -0,0 +1,94 @@
+---
+- name: Determine etcd version
+  hosts: etcd_hosts_to_upgrade
+  tasks:
+  - name: Record RPM based etcd version
+    command: rpm -qa --qf '%{version}' etcd\*
+    args:
+      warn: no
+    register: etcd_rpm_version
+    failed_when: false
+    when: not openshift.common.is_containerized | bool
+  - name: Record containerized etcd version
+    command: docker exec etcd_container rpm -qa --qf '%{version}' etcd\*
+    register: etcd_container_version
+    failed_when: false
+    when: openshift.common.is_containerized | bool
+
+# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
+# through hosts, then loop through tasks only when appropriate
+- name: Upgrade to 2.1
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.1'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_rpm_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+
+- name: Upgrade RPM hosts to 2.2
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.2'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_rpm_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+
+- name: Upgrade containerized hosts to 2.2.5
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: 2.2.5
+  tasks:
+  - include: containerized_tasks.yml
+    when: etcd_container_version.stdout | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool
+
+- name: Upgrade RPM hosts to 2.3
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.3'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_rpm_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+
+- name: Upgrade containerized hosts to 2.3.7
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: 2.3.7
+  tasks:
+  - include: containerized_tasks.yml
+    when: etcd_container_version.stdout | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool
+
+- name: Upgrade RPM hosts to 3.0
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '3.0'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_rpm_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+
+- name: Upgrade containerized hosts to etcd3 image
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: 3.0.14
+  tasks:
+  - include: containerized_tasks.yml
+    when: etcd_container_version.stdout | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool
+
+- name: Upgrade fedora to latest
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - include: fedora_tasks.yml
+    when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool
+
+- name: Backup etcd
+  include: backup.yml
+  vars:
+    backup_tag: "post-3.0-"
+  when: openshift_etcd_backup | default(true) | bool
diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
index dad80796c..474e6311e 100644
--- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
+++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
@@ -27,9 +27,8 @@
         embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
         debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level | default(2)) }}"
 
-- name: Backup etcd
-  include: ./etcd/backup.yml
-  when: not openshift_upgrade_skip_etcd_backup | default(false) | bool
+- name: Upgrade and backup etcd
+  include: ./etcd/main.yml
 
 - name: Upgrade master packages
   hosts: oo_masters_to_config