Add etcd upgrade for RHEL and Fedora

On Fedora we just blindly upgrade to the latest.
On RHEL we do stepwise upgrades 2.0,2.1,2.2,2.3,3.0

8 files changed, 138 insertions, 1 deletions

diff --git a/playbooks/common/openshift-cluster/upgrades/etcd_backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 57b156b1c..57b156b1c 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd_backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml
new file mode 100644
index 000000000..30232110e
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml
@@ -0,0 +1,23 @@
+---
+# F23 GA'd with etcd 2.0, currently has 2.2 in updates
+# F24 GA'd with etcd-2.2, currently has 2.2 in updates
+# F25 Beta currently has etcd 3.0
+- name: Verify cluster is healthy pre-upgrade
+  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+
+- name: Update etcd
+  package:
+    name: "etcd"
+    state: "latest"
+
+- name: Restart etcd
+  service:
+    name: etcd
+    state: restarted
+
+- name: Verify cluster is healthy
+  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins
new file mode 120000
index 000000000..27ddaa18b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins
@@ -0,0 +1 @@
+../../../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins
new file mode 120000
index 000000000..cf407f69b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins
@@ -0,0 +1 @@
+../../../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
new file mode 100644
index 000000000..e1c092dbe
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
@@ -0,0 +1,88 @@
+---
+# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to
+# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius
+# task for RHEL and CENTOS it's simply not possible in Fedora unless you've
+# mirrored packages on your own because only the GA and latest versions are
+# available in the repos. So for Fedora we'll simply skip this, sorry.
+
+- include: ../../evaluate_groups.yml
+  tags:
+  - always
+
+- name: Evaluate additional groups for upgrade
+  hosts: localhost
+  connection: local
+  become: no
+  tasks:
+  - name: Evaluate etcd_hosts_to_upgrade
+    add_host:
+      name: "{{ item }}"
+      groups: etcd_hosts_to_upgrade, etcd_hosts_to_backup
+    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}"
+
+- name: Backup etcd before upgrading anything
+  include: backup.yml
+  vars:
+    backup_tag: "pre-upgrade-"
+
+- name: Drop etcdctl profiles
+  hosts: etcd_hosts_to_upgrade
+  tasks:
+  - include: roles/etcd/tasks/etcdctl.yml
+
+- name: Determine etcd version
+  hosts: etcd_hosts_to_upgrade
+  tasks:
+  - name: Record etcd version
+    command: rpm -q --qf '%{version}' etcd
+    register: etcd_installed_version
+
+# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
+# through hosts, then loop through tasks only when appropriate
+- name: Upgrade to 2.1
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.1'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_installed_version.stdout | version_compare('2.1','<') and ansible_distribution == 'RedHat'
+
+- name: Upgrade to 2.2
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.2'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_installed_version.stdout | version_compare('2.2','<') and ansible_distribution == 'RedHat'
+
+- name: Upgrade to 2.3
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '2.3'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_installed_version.stdout | version_compare('2.3','<') and ansible_distribution == 'RedHat'
+
+- name: Upgrade to 3.0
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  vars:
+    upgrade_version: '3.0'
+  tasks:
+  - include: rhel_tasks.yml
+    when: etcd_installed_version.stdout | version_compare('3.0','<') and ansible_distribution == 'RedHat'
+
+- name: Upgrade fedora to latest
+  hosts: etcd_hosts_to_upgrade
+  serial: 1
+  tasks:
+  - include: fedora_tasks.yml
+    when: ansible_distribution == 'Fedora'
+
+- name: Backup etcd
+  include: backup.yml
+  vars:
+    backup_tag: "post-3.0-"
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml
new file mode 100644
index 000000000..8e7dc9d9b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml
@@ -0,0 +1,23 @@
+---
+- name: Verify cluster is healthy pre-upgrade
+  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+
+- name: Update etcd package but exclude etcd3
+  command: "{{ ansible_pkg_mgr }} install -y etcd-{{ upgrade_version }}\\* --exclude etcd3"
+  when: upgrade_version | version_compare('3.0','<')
+
+- name: Update etcd package not excluding etcd3
+  command: "{{ ansible_pkg_mgr }} install -y etcd3-{{ upgrade_version }}\\*"
+  when: not upgrade_version | version_compare('3.0','<')
+
+- name: Restart etcd
+  service:
+    name: etcd
+    state: restarted
+
+- name: Verify cluster is healthy
+  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/roles b/playbooks/common/openshift-cluster/upgrades/etcd/roles
new file mode 120000
index 000000000..6bc1a7aef
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/roles
@@ -0,0 +1 @@
+../../../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
index e1342f76b..339af3d0a 100644
--- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
+++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
@@ -27,7 +27,7 @@
         embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
 
 - name: Backup etcd
-  include: etcd_backup.yml
+  include: ./etcd/backup.yml
 
 - name: Upgrade master packages
   hosts: oo_masters_to_config