Merge pull request #6324 from mtnbikenc/consolidate-redeploy-certificates

Automatic merge from submit-queue.

Playbook Consolidation - Redeploy Certificates

This PR moves the certificate redeploy playbooks out of {byo,common}/openshift-cluster and into their respective component areas.

- playbooks/openshift-etcd (redeploy-certificates.yml, redeploy-ca.yml)
- playbooks/openshift-master (redeploy-certificates.yml, redeploy-openshift-ca.yml)
- playbooks/openshift-node (redeploy-certificates.yml)
- playbooks/openshift-hosted (redeploy-registry-certificates.yml, redeploy-router-certificates.yml)

playbooks/byo/openshift-cluster/redeploy-certificates.yml was moved to
playbooks/redeploy-certificates.yml

Trello: https://trello.com/c/zCz6RIHM/578-2-playbook-consolidation-openshift-cluster-redeploy-certificates

5 files changed, 354 insertions, 0 deletions

diff --git a/playbooks/openshift-master/private/certificates-backup.yml b/playbooks/openshift-master/private/certificates-backup.yml
new file mode 100644
index 000000000..4dbc041b0
--- /dev/null
+++ b/playbooks/openshift-master/private/certificates-backup.yml
@@ -0,0 +1,38 @@
+---
+- name: Backup and remove master cerftificates
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+    openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
+  pre_tasks:
+  - stat:
+      path: "{{ openshift.common.config_base }}/generated-configs"
+    register: openshift_generated_configs_dir_stat
+  - name: Backup generated certificate and config directories
+    command: >
+      tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz
+      {{ openshift.common.config_base }}/generated-configs
+      {{ openshift.common.config_base }}/master
+    when: openshift_generated_configs_dir_stat.stat.exists
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+  - name: Remove generated certificate directories
+    file:
+      path: "{{ item }}"
+      state: absent
+    with_items:
+    - "{{ openshift.common.config_base }}/generated-configs"
+  - name: Remove generated certificates
+    file:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+      state: absent
+    with_items:
+    - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false, include_ca=false) }}"
+    - "etcd.server.crt"
+    - "etcd.server.key"
+    - "master.server.crt"
+    - "master.server.key"
+    - "openshift-master.crt"
+    - "openshift-master.key"
+    - "openshift-master.kubeconfig"
diff --git a/playbooks/openshift-master/private/redeploy-certificates.yml b/playbooks/openshift-master/private/redeploy-certificates.yml
new file mode 100644
index 000000000..3bd38a61d
--- /dev/null
+++ b/playbooks/openshift-master/private/redeploy-certificates.yml
@@ -0,0 +1,6 @@
+---
+- include: certificates-backup.yml
+
+- include: certificates.yml
+  vars:
+    openshift_certificates_redeploy: true
diff --git a/playbooks/openshift-master/private/redeploy-openshift-ca.yml b/playbooks/openshift-master/private/redeploy-openshift-ca.yml
new file mode 100644
index 000000000..59657574a
--- /dev/null
+++ b/playbooks/openshift-master/private/redeploy-openshift-ca.yml
@@ -0,0 +1,300 @@
+---
+- name: Check cert expirys
+  hosts: oo_nodes_to_config:oo_masters_to_config:oo_etcd_to_config
+  vars:
+    openshift_certificate_expiry_show_all: yes
+  roles:
+  # Sets 'check_results' per host which contains health status for
+  # etcd, master and node certificates.  We will use 'check_results'
+  # to determine if any certificates were expired prior to running
+  # this playbook. Service restarts will be skipped if any
+  # certificates were previously expired.
+  - role: openshift_certificate_expiry
+
+# Update master config when ca-bundle not referenced. Services will be
+# restarted below after new CA certificate has been distributed.
+- name: Ensure ca-bundle.crt is referenced in master configuration
+  hosts: oo_masters_to_config
+  tasks:
+  - slurp:
+      src: "{{ openshift.common.config_base }}/master/master-config.yaml"
+    register: g_master_config_output
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: kubeletClientInfo.ca
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).kubeletClientInfo.ca != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: serviceAccountConfig.masterCA
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).serviceAccountConfig.masterCA != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: oauthConfig.masterCA
+      yaml_value: ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).oauthConfig.masterCA != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: etcdClientInfo.ca
+      yaml_value: ca-bundle.crt
+    when:
+    - groups.oo_etcd_to_config | default([]) | length == 0
+    - (g_master_config_output.content|b64decode|from_yaml).etcdClientInfo.ca != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: etcdConfig.peerServingInfo.clientCA
+      yaml_value: ca-bundle.crt
+    when:
+    - groups.oo_etcd_to_config | default([]) | length == 0
+    - (g_master_config_output.content|b64decode|from_yaml).etcdConfig.peerServingInfo.clientCA != 'ca-bundle.crt'
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: etcdConfig.servingInfo.clientCA
+      yaml_value: ca-bundle.crt
+    when:
+    - groups.oo_etcd_to_config | default([]) | length == 0
+    - (g_master_config_output.content|b64decode|from_yaml).etcdConfig.servingInfo.clientCA != 'ca-bundle.crt'
+  # Set servingInfo.clientCA to client-ca-bundle.crt in order to roll the CA certificate.
+  # This change will be reverted in playbooks/byo/openshift-cluster/redeploy-certificates.yml
+  - modify_yaml:
+      dest: "{{ openshift.common.config_base }}/master/master-config.yaml"
+      yaml_key: servingInfo.clientCA
+      yaml_value: client-ca-bundle.crt
+    when: (g_master_config_output.content|b64decode|from_yaml).servingInfo.clientCA != 'client-ca-bundle.crt'
+
+- name: Copy current OpenShift CA to legacy directory
+  hosts: oo_masters_to_config
+  pre_tasks:
+  - name: Create legacy-ca directory
+    file:
+      path: "{{ openshift.common.config_base }}/master/legacy-ca"
+      state: directory
+      mode: 0700
+      owner: root
+      group: root
+  - command: mktemp -u XXXXXX
+    register: g_legacy_ca_mktemp
+    changed_when: false
+  # Copy CA certificate, key, serial and bundle to legacy-ca with a
+  # prefix generated by mktemp, ie. XXXXXX-ca.crt.
+  #
+  # The following roles will pick up all CA certificates matching
+  # /.*-ca.crt/ in the legacy-ca directory and ensure they are present
+  # in the OpenShift CA bundle.
+  # - openshift_ca
+  # - openshift_master_certificates
+  # - openshift_node_certificates
+  - name: Copy current OpenShift CA to legacy directory
+    copy:
+      src: "{{ openshift.common.config_base }}/master/{{ item }}"
+      dest: "{{ openshift.common.config_base }}/master/legacy-ca/{{ g_legacy_ca_mktemp.stdout }}-{{ item }}"
+      remote_src: true
+    # It is possible that redeploying failed and files may be missing.
+    # Ignore errors in this case. Files should have been copied to
+    # legacy-ca directory in previous run.
+    ignore_errors: true
+    with_items:
+    - "ca.crt"
+    - "ca.key"
+    - "ca.serial.txt"
+    - "ca-bundle.crt"
+
+- name: Create temporary directory for creating new CA certificate
+  hosts: oo_first_master
+  tasks:
+  - name: Create temporary directory for creating new CA certificate
+    command: >
+      mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_new_openshift_ca_mktemp
+    changed_when: false
+
+- name: Create OpenShift CA
+  hosts: oo_first_master
+  vars:
+    # Set openshift_ca_config_dir to a temporary directory where CA
+    # will be created. We'll replace the existing CA with the CA
+    # created in the temporary directory.
+    openshift_ca_config_dir: "{{ hostvars[groups.oo_first_master.0].g_new_openshift_ca_mktemp.stdout }}"
+  roles:
+  - role: openshift_master_facts
+  - role: openshift_named_certificates
+  - role: openshift_ca
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: false
+
+- name: Retrieve OpenShift CA
+  hosts: oo_first_master
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - name: Retrieve CA certificate, key, bundle and serial
+    fetch:
+      src: "{{ hostvars[openshift_ca_host].g_new_openshift_ca_mktemp.stdout }}/{{ item }}"
+      dest: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items:
+    - ca.crt
+    - ca.key
+    - ca-bundle.crt
+    - ca.serial.txt
+    - client-ca-bundle.crt
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+    changed_when: false
+
+- name: Distribute OpenShift CA to masters
+  hosts: oo_masters_to_config
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - name: Deploy CA certificate, key, bundle and serial
+    copy:
+      src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/{{ item }}"
+      dest: "{{ openshift.common.config_base }}/master/"
+    with_items:
+    - ca.crt
+    - ca.key
+    - ca-bundle.crt
+    - ca.serial.txt
+    - client-ca-bundle.crt
+  - name: Update master client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/master/openshift-master.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
+  - name: Update admin client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
+  - name: Lookup default group for ansible_ssh_user
+    command: "/usr/bin/id -g {{ ansible_ssh_user | quote }}"
+    changed_when: false
+    register: _ansible_ssh_user_gid
+  - set_fact:
+      client_users: "{{ [ansible_ssh_user, 'root'] | unique }}"
+  - name: Create the client config dir(s)
+    file:
+      path: "~{{ item }}/.kube"
+      state: directory
+      mode: 0700
+      owner: "{{ item }}"
+      group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+    with_items: "{{ client_users }}"
+  - name: Copy the admin client config(s)
+    copy:
+      src: "{{ openshift.common.config_base }}/master/admin.kubeconfig"
+      dest: "~{{ item }}/.kube/config"
+      remote_src: yes
+    with_items: "{{ client_users }}"
+  - name: Update the permissions on the admin client config(s)
+    file:
+      path: "~{{ item }}/.kube/config"
+      state: file
+      mode: 0700
+      owner: "{{ item }}"
+      group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout  }}"
+    with_items: "{{ client_users }}"
+
+- include: restart.yml
+  # Do not restart masters when master or etcd certificates were previously expired.
+  when:
+  # masters
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_masters_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/master.server.crt"}))
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_masters_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/ca-bundle.crt"}))
+  # etcd
+  - ('expired' not in (hostvars
+      | oo_select_keys(groups['etcd'])
+      | oo_collect('check_results.check_results.etcd')
+      | oo_collect('health')))
+
+- name: Distribute OpenShift CA certificate to nodes
+  hosts: oo_nodes_to_config
+  vars:
+    openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+  tasks:
+  - copy:
+      src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/ca-bundle.crt"
+      dest: "{{ openshift.common.config_base }}/node/ca.crt"
+  - name: Copy OpenShift CA to system CA trust
+    copy:
+      src: "{{ item.cert }}"
+      dest: "/etc/pki/ca-trust/source/anchors/{{ item.id }}-{{ item.cert | basename }}"
+      remote_src: yes
+    with_items:
+    - id: openshift
+      cert: "{{ openshift.common.config_base }}/node/ca.crt"
+    notify:
+    - update ca trust
+  - name: Update node client kubeconfig CA data
+    kubeclient_ca:
+      client_path: "{{ openshift.common.config_base }}/node/system:node:{{ openshift.common.hostname }}.kubeconfig"
+      ca_path: "{{ openshift.common.config_base }}/node/ca.crt"
+  handlers:
+  # Normally this handler would restart docker after updating ca
+  # trust. We'll do that when we restart nodes to avoid restarting
+  # docker on all nodes in parallel.
+  - name: update ca trust
+    command: update-ca-trust
+
+- name: Delete temporary directory on CA host
+  hosts: oo_first_master
+  tasks:
+  - file:
+      path: "{{ g_new_openshift_ca_mktemp.stdout }}"
+      state: absent
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - file:
+      name: "{{ g_master_mktemp.stdout }}"
+      state: absent
+    changed_when: false
+
+- include: ../../openshift-node/private/restart.yml
+  # Do not restart nodes when node, master or etcd certificates were previously expired.
+  when:
+  # nodes
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_nodes_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_nodes_to_config.0].openshift.common.config_base ~ "/node/server.crt"}))
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_nodes_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_nodes_to_config.0].openshift.common.config_base ~ "/node/ca.crt"}))
+  # masters
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_masters_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/master.server.crt"}))
+  - ('expired' not in hostvars
+      | oo_select_keys(groups['oo_masters_to_config'])
+      | oo_collect('check_results.check_results.ocp_certs')
+      | oo_collect('health', {'path':hostvars[groups.oo_first_master.0].openshift.common.config_base ~ "/master/ca-bundle.crt"}))
+  # etcd
+  - ('expired' not in (hostvars
+      | oo_select_keys(groups['etcd'])
+      | oo_collect('check_results.check_results.etcd')
+      | oo_collect('health')))
diff --git a/playbooks/openshift-master/redeploy-certificates.yml b/playbooks/openshift-master/redeploy-certificates.yml
new file mode 100644
index 000000000..df727247b
--- /dev/null
+++ b/playbooks/openshift-master/redeploy-certificates.yml
@@ -0,0 +1,6 @@
+---
+- include: ../init/main.yml
+
+- include: private/redeploy-certificates.yml
+
+- include: private/restart.yml
diff --git a/playbooks/openshift-master/redeploy-openshift-ca.yml b/playbooks/openshift-master/redeploy-openshift-ca.yml
new file mode 100644
index 000000000..3ae74c7a0
--- /dev/null
+++ b/playbooks/openshift-master/redeploy-openshift-ca.yml
@@ -0,0 +1,4 @@
+---
+- include: ../init/main.yml
+
+- include: private/redeploy-openshift-ca.yml