Bug 1452939 - change Logging & Metrics imagePullPolicy

- all images logging and metrics change their default imagePullPolicy
  from Always to IfNotPresent

14 files changed, 164 insertions, 14 deletions

diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md
index 280d7d24c..0ea34faf2 100644
--- a/roles/openshift_logging/README.md
+++ b/roles/openshift_logging/README.md
@@ -225,3 +225,78 @@ The corresponding openshift\_logging\_mux\_* parameters are below.
 - `openshift_logging_mux_remote_syslog_tag_key`: If string specified, use this field from the record to set the key field on the syslog message
 - `openshift_logging_mux_remote_syslog_use_record`: Set `true` to use the severity and facility from the record, defaults to `false`
 - `openshift_logging_mux_remote_syslog_payload_key`: If string is specified, use this field from the record as the payload on the syslog message
+
+Image update procedure
+----------------------
+An upgrade of the logging stack from older version to newer is an automated process and should be performed by calling appropriate ansible playbook and setting required ansible variables in your inventory as documented in https://docs.openshift.org/.
+
+Following text describes manual update of the logging images without version upgrade. To determine the current version of images being used you can.
+```
+oc describe pod | grep 'Image ID:'
+```
+This will get the repo digest that can later be compared to the inspected image details.
+
+A way to determine when was your image last updated:
+```
+$ docker images
+REPOSITORY                              TAG     IMAGE ID       CREATED             SIZE
+<registry>/openshift3/logging-fluentd   v3.7    ff2e249fc45a   About an hour ago   235.2 MB
+
+$ docker inspect ff2e249fc45a
+[
+    {
+        . . .
+        "RepoDigests": [
+            "<registry>/openshift3/logging-fluentd@sha256:4346f0aa9694f32735115705ad324803b1a6ff08343c3288f7a62c3a5cb70495"
+        ],
+        . . .
+        "Config": {
+            . . .
+            "Labels": {
+                . . .
+                "build-date": "2017-10-12T14:38:22.414827",
+                . . . 
+                "release": "0.143.3.0",
+                . . .
+                "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/openshift3/logging-fluentd/images/v3.7.0-0.143.3.0",
+                . . .
+                "version": "v3.7.0"
+            }
+        },
+        . . .
+```
+
+Pull a new image to see if registry has any newer images with the same tag:
+```
+$ docker pull <registry>/openshift3/logging-fluentd:v3.7
+```
+
+If there was an update, you need to run the `docker pull` on each node.
+
+It is recommended that you now rerun the `openshift_logging` playbook to ensure that any necessary config changes are also picked up.
+ 
+To manually redeploy your pod you can do the following:
+- for a DC you can do:
+```
+oc rollout latest <dc_name>
+```
+     
+- for a RC you can scale down and scale back up
+```
+oc scale --replicas=0 <rc_name>
+
+... wait for scale down
+
+oc scale --replicas=<original_replica_count> <rc_name>
+```
+
+- for a DS you can delete the pod or unlabel and relabel your node
+```
+oc delete pod --selector=<ds_selector>
+```
+
+Changelog
+---------
+
+Tue Oct 10, 2017
+- Default imagePullPolicy changed from Always to IfNotPresent 
diff --git a/roles/openshift_logging/templates/jks_pod.j2 b/roles/openshift_logging/templates/jks_pod.j2
index 8b1c74211..e4110b7b3 100644
--- a/roles/openshift_logging/templates/jks_pod.j2
+++ b/roles/openshift_logging/templates/jks_pod.j2
@@ -8,7 +8,7 @@ spec:
   containers:
   - name: jks-cert-gen
     image: {{openshift_logging_image_prefix}}logging-deployer:{{openshift_logging_image_version}}
-    imagePullPolicy: Always
+    imagePullPolicy: IfNotPresent
     command: ["sh",  "{{generated_certs_dir}}/generate-jks.sh"]
     securityContext:
       privileged: true
diff --git a/roles/openshift_logging_curator/templates/curator.j2 b/roles/openshift_logging_curator/templates/curator.j2
index e71393643..462128366 100644
--- a/roles/openshift_logging_curator/templates/curator.j2
+++ b/roles/openshift_logging_curator/templates/curator.j2
@@ -38,7 +38,7 @@ spec:
         -
           name: "curator"
           image: {{image}}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
 {% if (curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "") or (curator_cpu_limit is defined and curator_cpu_limit is not none and curator_cpu_limit != "") or (curator_cpu_request is defined and curator_cpu_request is not none and curator_cpu_request != "") %}
           resources:
 {%   if (curator_memory_limit is defined and curator_memory_limit is not none and curator_memory_limit != "") or (curator_cpu_limit is defined and curator_cpu_limit is not none and curator_cpu_limit != "") %}
diff --git a/roles/openshift_logging_elasticsearch/templates/es.j2 b/roles/openshift_logging_elasticsearch/templates/es.j2
index 7966d219e..0c7d8b46e 100644
--- a/roles/openshift_logging_elasticsearch/templates/es.j2
+++ b/roles/openshift_logging_elasticsearch/templates/es.j2
@@ -41,7 +41,7 @@ spec:
       containers:
         - name: proxy
           image: {{ proxy_image }}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           args:
            - --upstream-ca=/etc/elasticsearch/secret/admin-ca
            - --https-address=:4443
@@ -76,7 +76,7 @@ spec:
         -
           name: "elasticsearch"
           image: {{image}}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           resources:
             limits:
 {% if es_cpu_limit is defined and es_cpu_limit is not none and es_cpu_limit != '' %}
diff --git a/roles/openshift_logging_eventrouter/files/eventrouter-template.yaml b/roles/openshift_logging_eventrouter/files/eventrouter-template.yaml
index 91708e54b..cc01c010d 100644
--- a/roles/openshift_logging_eventrouter/files/eventrouter-template.yaml
+++ b/roles/openshift_logging_eventrouter/files/eventrouter-template.yaml
@@ -56,7 +56,7 @@ objects:
           containers:
             - name: kube-eventrouter
               image: ${IMAGE}
-              imagePullPolicy: Always
+              imagePullPolicy: IfNotPresent
               resources:
                 limits:
                   memory: ${MEMORY} 
diff --git a/roles/openshift_logging_eventrouter/templates/eventrouter-template.j2 b/roles/openshift_logging_eventrouter/templates/eventrouter-template.j2
index 7fdf959d3..5a4f7f762 100644
--- a/roles/openshift_logging_eventrouter/templates/eventrouter-template.j2
+++ b/roles/openshift_logging_eventrouter/templates/eventrouter-template.j2
@@ -62,7 +62,7 @@ objects:
           containers:
             - name: kube-eventrouter
               image: ${IMAGE}
-              imagePullPolicy: Always
+              imagePullPolicy: IfNotPresent
               resources:
                 limits:
                   memory: ${MEMORY}
diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2
index b07175a50..10283316c 100644
--- a/roles/openshift_logging_fluentd/templates/fluentd.j2
+++ b/roles/openshift_logging_fluentd/templates/fluentd.j2
@@ -29,7 +29,7 @@ spec:
       containers:
       - name: "{{ daemonset_container_name }}"
         image: "{{ openshift_logging_fluentd_image_prefix }}{{ daemonset_name }}:{{ openshift_logging_fluentd_image_version }}"
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         securityContext:
           privileged: true
 {% if (fluentd_memory_limit is defined and fluentd_memory_limit is not none) or (fluentd_cpu_limit is defined and fluentd_cpu_limit is not none) or (fluentd_cpu_request is defined and fluentd_cpu_request is not none) %}
diff --git a/roles/openshift_logging_kibana/templates/kibana.j2 b/roles/openshift_logging_kibana/templates/kibana.j2
index 329ccbde2..4ff86729a 100644
--- a/roles/openshift_logging_kibana/templates/kibana.j2
+++ b/roles/openshift_logging_kibana/templates/kibana.j2
@@ -37,7 +37,7 @@ spec:
         -
           name: "kibana"
           image: {{ image }}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
 {% if (kibana_memory_limit is defined and kibana_memory_limit is not none and kibana_memory_limit != "") or (kibana_cpu_limit is defined and kibana_cpu_limit is not none and kibana_cpu_limit != "") or (kibana_cpu_request is defined and kibana_cpu_request is not none and kibana_cpu_request != "") %}
           resources:
 {%   if (kibana_memory_limit is defined and kibana_memory_limit is not none and kibana_memory_limit != "") or (kibana_cpu_limit is defined and kibana_cpu_limit is not none and kibana_cpu_limit != "") %}
@@ -84,7 +84,7 @@ spec:
         -
           name: "kibana-proxy"
           image: {{ proxy_image }}
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
 {% if (kibana_proxy_memory_limit is defined and kibana_proxy_memory_limit is not none and kibana_proxy_memory_limit != "") or (kibana_proxy_cpu_limit is defined and kibana_proxy_cpu_limit is not none and kibana_proxy_cpu_limit != "") or (kibana_proxy_cpu_request is defined and kibana_proxy_cpu_request is not none and kibana_proxy_cpu_request != "") %}
           resources:
 {%   if (kibana_proxy_memory_limit is defined and kibana_proxy_memory_limit is not none and kibana_proxy_memory_limit != "") or (kibana_proxy_cpu_limit is defined and kibana_proxy_cpu_limit is not none and kibana_proxy_cpu_limit != "") %}
diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2
index 7e88e3964..cfb13d59b 100644
--- a/roles/openshift_logging_mux/templates/mux.j2
+++ b/roles/openshift_logging_mux/templates/mux.j2
@@ -36,7 +36,7 @@ spec:
       containers:
       - name: "mux"
         image: {{image}}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
 {% if (mux_memory_limit is defined and mux_memory_limit is not none) or (mux_cpu_limit is defined and mux_cpu_limit is not none) or (mux_cpu_request is defined and mux_cpu_request is not none) %}
         resources:
 {%   if (mux_memory_limit is defined and mux_memory_limit is not none) or (mux_cpu_limit is defined and mux_cpu_limit is not none) %}
diff --git a/roles/openshift_metrics/README.md b/roles/openshift_metrics/README.md
index ed698daca..b74f22c00 100644
--- a/roles/openshift_metrics/README.md
+++ b/roles/openshift_metrics/README.md
@@ -109,3 +109,78 @@ Author Information
 ------------------
 
 Jose David Martín (j.david.nieto@gmail.com)
+
+Image update procedure
+----------------------
+An upgrade of the metrics stack from older version to newer is an automated process and should be performed by calling appropriate ansible playbook and setting required ansible variables in your inventory as documented in https://docs.openshift.org/.
+
+Following text describes manual update of the metrics images without version upgrade. To determine the current version of images being used you can:
+```
+oc describe pod | grep 'Image ID:'
+```
+This will get the repo digest that can later be compared to the inspected image details.
+
+A way to determine when was your image last updated:
+```
+$ docker images
+REPOSITORY                                       TAG     IMAGE ID       CREATED             SIZE
+<registry>/openshift3/origin-metrics-cassandra   v3.7    f8ad8d569e27   14 hours ago        783.7 MB
+
+$ docker inspect 9c3597aeb39f 
+[
+    {
+        . . .
+        "RepoDigests": [
+            "<registry>/openshift3/metrics-cassandra@sha256:d37fc0cab268625b53a92bb98d09fcc501cfca1c68e16bac6dd98446d32ba135
+        ],
+        . . .
+        "Config": {
+            . . .
+            "Labels": {
+                . . .
+                "build-date": "2017-10-17T16:47:44.350655",
+                . . . 
+                "release": "0.143.4.0",
+                . . .
+                "url": "https://access.redhat.com/containers/#/registry.access.redhat.com/openshift3/metrics-cassandra/images/v3.7.0-0.143.4.0",
+                . . .
+                "version": "v3.7.0"
+            }
+        },
+        . . .
+```
+
+Pull a new image to see if registry has any newer images with the same tag:
+```
+$ docker pull <registry>/openshift3/origin-metrics-cassandra:v3.7
+```
+
+If there was an update, you need to run the `docker pull` on each node.
+
+It is recommended that you now rerun the `openshift_metrics` playbook to ensure that any necessary config changes are also picked up.
+ 
+To manually redeploy your pod you can do the following:
+- for a DC you can do:
+```
+oc rollout latest <dc_name>
+```
+     
+- for a RC you can scale down and scale back up
+```
+oc scale --replicas=0 <rc_name>
+
+... wait for scale down
+
+oc scale --replicas=<original_replica_count> <rc_name>
+```
+
+- for a DS you can delete the pod or unlabel and relabel your node
+```
+oc delete pod --selector=<ds_selector>
+```
+
+Changelog
+---------
+
+Tue Oct 10, 2017
+- Default imagePullPolicy changed from Always to IfNotPresent 
diff --git a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
index 6f341bcfb..6a3811598 100644
--- a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
@@ -30,7 +30,7 @@ spec:
 {% endif %}
       containers:
       - image: "{{ openshift_metrics_image_prefix }}metrics-cassandra:{{ openshift_metrics_image_version }}"
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: hawkular-cassandra-{{ node }}
         ports:
         - name: cql-port
diff --git a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
index 59f7fb44a..0662bea53 100644
--- a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
@@ -25,7 +25,7 @@ spec:
 {% endif %}
       containers:
       - image: {{openshift_metrics_image_prefix}}metrics-hawkular-metrics:{{openshift_metrics_image_version}}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: hawkular-metrics
         ports:
         - name: http-endpoint
diff --git a/roles/openshift_metrics/templates/hawkular_openshift_agent_ds.j2 b/roles/openshift_metrics/templates/hawkular_openshift_agent_ds.j2
index d65eaf9ae..40d09e9fa 100644
--- a/roles/openshift_metrics/templates/hawkular_openshift_agent_ds.j2
+++ b/roles/openshift_metrics/templates/hawkular_openshift_agent_ds.j2
@@ -25,7 +25,7 @@ spec:
 {% endif %}
       containers:
       - image: {{openshift_metrics_image_prefix}}metrics-hawkular-openshift-agent:{{openshift_metrics_image_version}}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         name: hawkular-openshift-agent
 {% if ((openshift_metrics_hawkular_agent_limits_cpu is defined and openshift_metrics_hawkular_agent_limits_cpu is not none)
    or (openshift_metrics_hawkular_agent_limits_memory is defined and openshift_metrics_hawkular_agent_limits_memory is not none)
diff --git a/roles/openshift_metrics/templates/heapster.j2 b/roles/openshift_metrics/templates/heapster.j2
index d8c7763ea..e732c1eee 100644
--- a/roles/openshift_metrics/templates/heapster.j2
+++ b/roles/openshift_metrics/templates/heapster.j2
@@ -27,7 +27,7 @@ spec:
       containers:
       - name: heapster
         image: {{openshift_metrics_image_prefix}}metrics-heapster:{{openshift_metrics_image_version}}
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8082
           name: "http-endpoint"