Merge pull request #2285 from dgoodwin/nuke-images-2

Improvements for Docker 1.10+ Upgrade Image Nuking

4 files changed, 59 insertions, 20 deletions

diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
index 20d66522f..03e7b844c 100644
--- a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
@@ -13,14 +13,32 @@
   failed_when: false
   when: openshift.common.is_containerized | bool
 
+- name: Check Docker image count
+  shell: "docker images -aq | wc -l"
+  register: docker_image_count
+
+- debug: var=docker_image_count.stdout
+
 - name: Remove all containers and images
   script: nuke_images.sh docker
   register: nuke_images_result
   when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
 
+- name: Check Docker image count
+  shell: "docker images -aq | wc -l"
+  register: docker_image_count
+  when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+- debug: var=docker_image_count.stdout
+  when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+- service: name=docker state=stopped
+
 - name: Upgrade Docker
   action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version }} state=present"
 
+- service: name=docker state=started
+
 - name: Restart containerized services
   service: name={{ item }} state=started
   with_items:
diff --git a/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh b/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh
index 6b155f7fa..8635eab0d 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh
+++ b/playbooks/common/openshift-cluster/upgrades/files/nuke_images.sh
@@ -15,9 +15,11 @@ then
 fi
 
 # Delete all images (forcefully)
-image_ids=`docker images -q`
+image_ids=`docker images -aq`
 if test -n "$image_ids"
 then
-    # Taken from: https://gist.github.com/brianclements/f72b2de8e307c7b56689#gistcomment-1443144
-    docker rmi $(docker images | grep "$2/\|/$2 \| $2 \|$2 \|$2-\|$2_" | awk '{print $1 ":" $2}') 2>/dev/null || echo "No images matching \"$2\" left to purge."
+    # Some layers are deleted recursively and are no longer present
+    # when docker goes to remove them:
+    docker rmi -f `docker images -aq` || true
 fi
+
diff --git a/playbooks/common/openshift-cluster/upgrades/pre.yml b/playbooks/common/openshift-cluster/upgrades/pre.yml
index b5fbc4af6..42a24eaf8 100644
--- a/playbooks/common/openshift-cluster/upgrades/pre.yml
+++ b/playbooks/common/openshift-cluster/upgrades/pre.yml
@@ -185,10 +185,12 @@
 - name: Verify docker upgrade targets
   hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
   tasks:
-  - name: Determine available Docker
-    script: ../files/rpm_versions.sh docker
-    register: g_docker_version_result
-    when: not openshift.common.is_atomic | bool
+  # Only check if docker upgrade is required if docker_upgrade is not
+  # already set to False.
+  - include: docker/upgrade_check.yml
+    when: docker_upgrade is not defined or docker_upgrade | bool and not openshift.common.is_atomic | bool
+
+  # Additional checks for Atomic hosts:
 
   - name: Determine available Docker
     shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker"
@@ -196,18 +198,12 @@
     when: openshift.common.is_atomic | bool
 
   - set_fact:
-      g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}"
-    when: not openshift.common.is_atomic | bool
-
-  - set_fact:
-      g_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}"
+      l_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}"
     when: openshift.common.is_atomic | bool
 
   - fail:
       msg: This playbook requires access to Docker 1.10 or later
-    when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.10','<')
-
-  # TODO: add check to upgrade ostree to get latest Docker
+    when: openshift.common.is_atomic | bool and l_docker_version.avail_version | default(l_docker_version.curr_version, true) | version_compare('1.10','<')
 
   - set_fact:
       pre_upgrade_complete: True
diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/upgrade.yml
index dee086cf5..3ec47d6f3 100644
--- a/playbooks/common/openshift-cluster/upgrades/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/upgrade.yml
@@ -3,6 +3,34 @@
 # The restart playbook should be run after this playbook completes.
 ###############################################################################
 
+# Separate step so we can execute in parallel and clear out anything unused
+# before we get into the serialized upgrade process which will then remove
+# remaining images if possible.
+- name: Cleanup unused Docker images
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
+  tasks:
+  - name: Check Docker image count
+    shell: "docker images -aq | wc -l"
+    register: docker_image_count
+    when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+  - debug: var=docker_image_count.stdout
+    when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+  - name: Remove unused Docker images for Docker 1.10+ migration
+    shell: "docker rmi `docker images -aq`"
+    # Will fail on images still in use:
+    failed_when: false
+    when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+  - name: Check Docker image count
+    shell: "docker images -aq | wc -l"
+    register: docker_image_count
+    when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+  - debug: var=docker_image_count.stdout
+    when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
 ###############################################################################
 # Upgrade Masters
 ###############################################################################
@@ -111,11 +139,6 @@
     delegate_to: "{{ groups.oo_first_master.0 }}"
     when: inventory_hostname in groups.oo_nodes_to_config
 
-  # Only check if docker upgrade is required if docker_upgrade is not
-  # already set to False.
-  - include: docker/upgrade_check.yml
-    when: docker_upgrade is not defined or docker_upgrade | bool and not openshift.common.is_atomic | bool
-
   - include: docker/upgrade.yml
     when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool
   - include: "{{ node_config_hook }}"