Merge pull request #1155 from mwoodson/host_monitoring

merging master into prod

17 files changed, 113 insertions, 33 deletions

diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index 482fa8441..11e5b68f6 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -1,8 +1,12 @@
 ---
 - include: evaluate_groups.yml
 
+- include: ../openshift-docker/config.yml
+
 - include: ../openshift-etcd/config.yml
 
+- include: ../openshift-nfs/config.yml
+
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-node/config.yml
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index 6343a2567..db7105ed5 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -21,6 +21,14 @@
       msg: This playbook requires g_lb_hosts to be set
     when: g_lb_hosts is not defined
 
+  - fail:
+      msg: This playbook requires g_nfs_hosts to be set
+    when: g_nfs_hosts is not defined
+
+  - fail:
+      msg: The nfs group must be limited to one host
+    when: (groups[g_nfs_hosts] | default([])) | length > 1
+
   - name: Evaluate oo_etcd_to_config
     add_host:
       name: "{{ item }}"
@@ -81,3 +89,11 @@
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
     with_items: "{{ g_lb_hosts | default([]) }}"
+
+  - name: Evaluate oo_nfs_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nfs_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: "{{ g_nfs_hosts | default([]) }}"
diff --git a/playbooks/common/openshift-cluster/update_repos_and_packages.yml b/playbooks/common/openshift-cluster/update_repos_and_packages.yml
index 190e2d862..88736ee03 100644
--- a/playbooks/common/openshift-cluster/update_repos_and_packages.yml
+++ b/playbooks/common/openshift-cluster/update_repos_and_packages.yml
@@ -4,9 +4,10 @@
     openshift_deployment_type: "{{ deployment_type }}"
   roles:
   - role: rhel_subscribe
-    when: deployment_type == "enterprise" and
+    when: deployment_type in ["enterprise", "atomic-enterprise", "openshift-enterprise"] and
           ansible_distribution == "RedHat" and
           lookup('oo_option', 'rhel_skip_subscription') | default(rhsub_skip, True) |
             default('no', True) | lower in ['no', 'false']
+          and not openshift.common.is_atomic | bool
   - openshift_repos
   - os_update_latest
diff --git a/playbooks/common/openshift-cluster/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
index c7c966b60..b46407ed7 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/versions.sh
+++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
@@ -1,9 +1,8 @@
 #!/bin/bash
 
-yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
-
-yum_available=$(yum list available -q "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
+yum_installed=$(yum list installed -e 0 -q "$@" 2>&1 | tail -n +2 | awk '{ print $2 }' | tr '\n' ' ')
 
+yum_available=$(yum list available -e 0 -q "$@" 2>&1 | tail -n +2 | grep -v 'el7ose' | awk '{ print $2 }' | tr '\n' ' ')
 
 echo "---"
 echo "curr_version: ${yum_installed}"
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index fc098b4ed..68df2153d 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -29,6 +29,7 @@
   hosts: oo_first_master
   vars:
     openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    target_version: "{{ '1.1' if deployment_type == 'origin' else '3.1' }}"
   gather_facts: no
   tasks:
   # Pacemaker is currently the only supported upgrade path for multiple masters
@@ -45,8 +46,8 @@
   - fail:
       msg: >
         openshift_pkg_version is {{ openshift_pkg_version }} which is not a
-        valid version for a 3.1 upgrade
-    when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare('3.0.2.900','<')
+        valid version for a {{ target_version }} upgrade
+    when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(target_version ,'<')
 
   # If this script errors out ansible will show the default stdout/stderr
   # which contains details for the user:
@@ -55,6 +56,8 @@
 
 - name: Verify upgrade can proceed
   hosts: oo_masters_to_config:oo_nodes_to_config
+  vars:
+    target_version: "{{ '1.1' if deployment_type == 'origin' else '3.1' }}"
   tasks:
   - name: Clean package cache
     command: "{{ ansible_pkg_mgr }} clean all"
@@ -77,8 +80,8 @@
     when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
 
   - fail:
-      msg: Atomic OpenShift 3.1 packages not found
-    when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
+      msg: Upgrade packages not found
+    when: (g_aos_versions.avail_version | default(g_aos_versions.curr_version, true) | version_compare(target_version, '<'))
 
   - set_fact:
       pre_upgrade_complete: True
@@ -235,6 +238,7 @@
 
   - name: Ensure python-yaml present for config upgrade
     action: "{{ ansible_pkg_mgr }} name=PyYAML state=present"
+    when: not openshift.common.is_atomic | bool
 
   - name: Upgrade master configuration
     openshift_upgrade_config:
diff --git a/playbooks/common/openshift-docker/config.yml b/playbooks/common/openshift-docker/config.yml
new file mode 100644
index 000000000..092d5533c
--- /dev/null
+++ b/playbooks/common/openshift-docker/config.yml
@@ -0,0 +1,9 @@
+- name: Configure docker hosts
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
+  vars:
+    docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}"
+    docker_insecure_registries: "{{ lookup('oo_option',  'docker_insecure_registries') | oo_split }}"
+    docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}"
+  roles:
+  - openshift_facts
+  - openshift_docker
diff --git a/playbooks/common/openshift-docker/filter_plugins b/playbooks/common/openshift-docker/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/common/openshift-docker/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-docker/lookup_plugins b/playbooks/common/openshift-docker/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-docker/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-docker/roles b/playbooks/common/openshift-docker/roles
new file mode 120000
index 000000000..20c4c58cf
--- /dev/null
+++ b/playbooks/common/openshift-docker/roles
@@ -0,0 +1 @@
+../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 6dee196e3..9a5ae0e6b 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -14,7 +14,8 @@
           public_hostname: "{{ openshift_public_hostname | default(None) }}"
           deployment_type: "{{ openshift_deployment_type }}"
       - role: etcd
-        local_facts: {}
+        local_facts:
+          etcd_image: "{{ osm_etcd_image | default(None) }}"
   - name: Check status of etcd certificates
     stat:
       path: "{{ item }}"
@@ -88,6 +89,7 @@
   roles:
   - etcd
   - role: nickhammond.logrotate
+    when: not openshift.common.is_containerized | bool
 
 - name: Delete temporary directory on localhost
   hosts: localhost
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index dd638487a..677c274c4 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -232,31 +232,36 @@
       balance: source
       servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
   roles:
+  - role: openshift_facts
   - role: haproxy
     when: groups.oo_masters_to_config | length > 1
 
-- name: Generate master session keys
+- name: Check for cached session secrets
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+          session_auth_secrets: "{{ openshift_master_session_auth_secrets | default(openshift.master.session_auth_secrets | default(None)) }}"
+          session_encryption_secrets: "{{ openshift_master_session_encryption_secrets | default(openshift.master.session_encryption_secrets | default(None)) }}"
+
+- name: Generate master session secrets
   hosts: oo_first_master
+  vars:
+    g_session_secrets_present: "{{ (openshift.master.session_auth_secrets | default([]) and openshift.master.session_encryption_secrets | default([])) | length > 0 }}"
+    g_session_auth_secrets: "{{ [ 24 | oo_generate_secret ] }}"
+    g_session_encryption_secrets: "{{ [ 24 | oo_generate_secret ] }}"
+  roles:
+  - role: openshift_facts
   tasks:
-  - fail:
-      msg: "Both openshift_master_session_auth_secrets and openshift_master_session_encryption_secrets must be provided if either variable is set"
-    when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is not defined) or (openshift_master_session_encryption_secrets is defined and openshift_master_session_auth_secrets is not defined)
-  - fail:
-      msg: "openshift_master_session_auth_secrets and openshift_master_encryption_secrets must be equal length"
-    when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is defined) and (openshift_master_session_auth_secrets | length != openshift_master_session_encryption_secrets | length)
-  - name: Install OpenSSL package
-    action: "{{ ansible_pkg_mgr }} name=openssl state=present"
-  - name: Generate session authentication key
-    command: /usr/bin/openssl rand -base64 24
-    register: session_auth_output
-    when: openshift_master_session_auth_secrets is undefined
-  - name: Generate session encryption key
-    command: /usr/bin/openssl rand -base64 24
-    register: session_encryption_output
-    when: openshift_master_session_encryption_secrets is undefined
-  - set_fact:
-      session_auth_secret: "{{ openshift_master_session_auth_secrets | default([session_auth_output.stdout]) }}"
-      session_encryption_secret: "{{ openshift_master_session_encryption_secrets | default([session_encryption_output.stdout]) }}"
+  - openshift_facts:
+      role: master
+      local_facts:
+        session_auth_secrets: "{{ g_session_auth_secrets }}"
+        session_encryption_secrets: "{{ g_session_encryption_secrets }}"
+    when: not g_session_secrets_present | bool
 
 - name: Parse named certificates
   hosts: localhost
@@ -312,8 +317,8 @@
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
     openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
-    openshift_master_session_auth_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_auth_secret'] }}"
-    openshift_master_session_encryption_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_encryption_secret'] }}"
+    openshift_master_session_auth_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_auth_secrets }}"
+    openshift_master_session_encryption_secrets: "{{ hostvars[groups.oo_first_master.0].openshift.master.session_encryption_secrets }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -328,6 +333,7 @@
   roles:
   - openshift_master
   - role: nickhammond.logrotate
+    when: not openshift.common.is_containerized | bool
   - role: fluentd_master
     when: openshift.common.use_fluentd | bool
   post_tasks:
@@ -356,7 +362,7 @@
     cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
   roles:
   - role: cockpit
-    when: ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
+    when: not openshift.common.is_containerized and ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
       (osm_use_cockpit | bool or osm_use_cockpit is undefined )
 
 - name: Configure flannel
@@ -394,7 +400,15 @@
 
 - name: Create services
   hosts: oo_first_master
+  vars:
+    attach_registry_volume: "{{ groups.oo_nfs_to_config | length > 0 }}"
+  pre_tasks:
+  - set_fact:
+      nfs_host: "{{ groups.oo_nfs_to_config.0 }}"
+      registry_volume_path: "{{ hostvars[groups.oo_nfs_to_config.0].openshift.nfs.exports_dir + '/' + hostvars[groups.oo_nfs_to_config.0].openshift.nfs.registry_volume }}"
+    when: attach_registry_volume | bool
   roles:
   - role: openshift_router
     when: openshift.master.infra_nodes is defined
-  #- role: openshift_registry
+  - role: openshift_registry
+    when: openshift.master.infra_nodes is defined and attach_registry_volume | bool
diff --git a/playbooks/common/openshift-nfs/config.yml b/playbooks/common/openshift-nfs/config.yml
new file mode 100644
index 000000000..e3f5c17ca
--- /dev/null
+++ b/playbooks/common/openshift-nfs/config.yml
@@ -0,0 +1,5 @@
+---
+- name: Configure nfs hosts
+  hosts: oo_nfs_to_config
+  roles:
+  - role: openshift_storage_nfs
diff --git a/playbooks/common/openshift-nfs/filter_plugins b/playbooks/common/openshift-nfs/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/common/openshift-nfs/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-nfs/lookup_plugins b/playbooks/common/openshift-nfs/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-nfs/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-nfs/roles b/playbooks/common/openshift-nfs/roles
new file mode 120000
index 000000000..e2b799b9d
--- /dev/null
+++ b/playbooks/common/openshift-nfs/roles
@@ -0,0 +1 @@
+../../../roles/
\ No newline at end of file
diff --git a/playbooks/common/openshift-nfs/service.yml b/playbooks/common/openshift-nfs/service.yml
new file mode 100644
index 000000000..20c8ca248
--- /dev/null
+++ b/playbooks/common/openshift-nfs/service.yml
@@ -0,0 +1,18 @@
+---
+- name: Populate g_service_nfs host group if needed
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - fail: msg="new_cluster_state is required to be injected in this playbook"
+    when: new_cluster_state is not defined
+
+  - name: Evaluate g_service_nfs
+    add_host: name={{ item }} groups=g_service_nfs
+    with_items: oo_host_group_exp | default([])
+
+- name: Change state on nfs instance(s)
+  hosts: g_service_nfs
+  connection: ssh
+  gather_facts: no
+  tasks:
+    - service: name=nfs-server state="{{ new_cluster_state }}"
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 69ccb0cb8..483a7768c 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -16,6 +16,7 @@
           hostname: "{{ openshift_hostname | default(None) }}"
           public_hostname: "{{ openshift_public_hostname | default(None) }}"
           deployment_type: "{{ openshift_deployment_type }}"
+          use_flannel: "{{ openshift_use_flannel | default(None) }}"
       - role: node
         local_facts:
           labels: "{{ openshift_node_labels | default(None) }}"
@@ -181,6 +182,7 @@
   - role: flannel
     when: openshift.common.use_flannel | bool
   - role: nickhammond.logrotate
+    when: not openshift.common.is_containerized | bool
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool
   tasks: