diff options
80 files changed, 1726 insertions, 548 deletions
diff --git a/callback_plugins/aa_version_requirement.py b/callback_plugins/aa_version_requirement.py index 40affb18b..f31445381 100644 --- a/callback_plugins/aa_version_requirement.py +++ b/callback_plugins/aa_version_requirement.py @@ -7,6 +7,7 @@ The plugin is named with leading `aa_` to ensure this plugin is loaded first (alphanumerically) by Ansible. """ import sys +from subprocess import check_output from ansible import __version__ if __version__ < '2.0': @@ -65,7 +66,11 @@ class CallbackModule(CallbackBase): sys.exit(1) if __version__ == '2.2.1.0': - display( - 'FATAL: Current Ansible version (%s) is not supported. %s' - % (__version__, FAIL_ON_2_2_1_0), color='red') - sys.exit(1) + rpm_ver = str(check_output(["rpm", "-qa", "ansible"])) + patched_ansible = '2.2.1.0-2' + + if patched_ansible not in rpm_ver: + display( + 'FATAL: Current Ansible version (%s) is not supported. %s' + % (__version__, FAIL_ON_2_2_1_0), color='red') + sys.exit(1) diff --git a/filter_plugins/openshift_master.py b/filter_plugins/openshift_master.py index f209d6c3b..77b0a3dc9 100644 --- a/filter_plugins/openshift_master.py +++ b/filter_plugins/openshift_master.py @@ -517,13 +517,11 @@ class FilterModule(object): return valid @staticmethod - def certificates_to_synchronize(hostvars, include_keys=True): + def certificates_to_synchronize(hostvars, include_keys=True, include_ca=True): ''' Return certificates to synchronize based on facts. ''' if not issubclass(type(hostvars), dict): raise errors.AnsibleFilterError("|failed expects hostvars is a dict") - certs = ['ca.crt', - 'ca.key', - 'admin.crt', + certs = ['admin.crt', 'admin.key', 'admin.kubeconfig', 'master.kubelet-client.crt', @@ -534,6 +532,8 @@ class FilterModule(object): 'openshift-router.crt', 'openshift-router.key', 'openshift-router.kubeconfig'] + if bool(include_ca): + certs += ['ca.crt', 'ca.key'] if bool(include_keys): certs += ['serviceaccounts.private.key', 'serviceaccounts.public.key'] diff --git a/inventory/byo/hosts.origin.example b/inventory/byo/hosts.origin.example index 0a1b8c5c4..f24cfc737 100644 --- a/inventory/byo/hosts.origin.example +++ b/inventory/byo/hosts.origin.example @@ -79,7 +79,7 @@ openshift_release=v1.4 # Disable pushing to dockerhub #openshift_docker_disable_push_dockerhub=True # Items added, as is, to end of /etc/sysconfig/docker OPTIONS -# Default value: "--log-driver=json-file --log-opt max-size=50m" +# Default value: "--log-driver=journald" #openshift_docker_options="-l warn --ipv6=false" # Specify exact version of Docker to configure or upgrade to. @@ -528,10 +528,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', # NOTE: CA certificate will not be replaced with existing clusters. # This option may only be specified when creating a new cluster or # when redeploying cluster certificates with the redeploy-certificates -# playbook. If replacing the CA certificate in an existing cluster -# with a custom ca certificate, the following variable must also be -# set. -#openshift_certificates_redeploy_ca=true +# playbook. # Configure custom named certificates (SNI certificates) # diff --git a/inventory/byo/hosts.ose.example b/inventory/byo/hosts.ose.example index 89b9d7e48..b48776304 100644 --- a/inventory/byo/hosts.ose.example +++ b/inventory/byo/hosts.ose.example @@ -79,7 +79,7 @@ openshift_release=v3.4 # Disable pushing to dockerhub #openshift_docker_disable_push_dockerhub=True # Items added, as is, to end of /etc/sysconfig/docker OPTIONS -# Default value: "--log-driver=json-file --log-opt max-size=50m" +# Default value: "--log-driver=journald" #openshift_docker_options="-l warn --ipv6=false" # Specify exact version of Docker to configure or upgrade to. @@ -528,10 +528,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', # NOTE: CA certificate will not be replaced with existing clusters. # This option may only be specified when creating a new cluster or # when redeploying cluster certificates with the redeploy-certificates -# playbook. If replacing the CA certificate in an existing cluster -# with a custom ca certificate, the following variable must also be -# set. -#openshift_certificates_redeploy_ca=true +# playbook. # Configure custom named certificates (SNI certificates) # diff --git a/library/kubeclient_ca.py b/library/kubeclient_ca.py new file mode 100644 index 000000000..163624a76 --- /dev/null +++ b/library/kubeclient_ca.py @@ -0,0 +1,90 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# vim: expandtab:tabstop=4:shiftwidth=4 + +''' kubeclient_ca ansible module ''' + +import base64 +import yaml +from ansible.module_utils.basic import AnsibleModule + + +DOCUMENTATION = ''' +--- +module: kubeclient_ca +short_description: Modify kubeclient certificate-authority-data +author: Andrew Butcher +requirements: [ ] +''' +EXAMPLES = ''' +- kubeclient_ca: + client_path: /etc/origin/master/admin.kubeconfig + ca_path: /etc/origin/master/ca-bundle.crt + +- slurp: + src: /etc/origin/master/ca-bundle.crt + register: ca_data +- kubeclient_ca: + client_path: /etc/origin/master/admin.kubeconfig + ca_data: "{{ ca_data.content }}" +''' + + +def main(): + ''' Modify kubeconfig located at `client_path`, setting the + certificate authority data to specified `ca_data` or contents of + `ca_path`. + ''' + + module = AnsibleModule( # noqa: F405 + argument_spec=dict( + client_path=dict(required=True), + ca_data=dict(required=False, default=None), + ca_path=dict(required=False, default=None), + backup=dict(required=False, default=True, type='bool'), + ), + supports_check_mode=True, + mutually_exclusive=[['ca_data', 'ca_path']], + required_one_of=[['ca_data', 'ca_path']] + ) + + client_path = module.params['client_path'] + ca_data = module.params['ca_data'] + ca_path = module.params['ca_path'] + backup = module.params['backup'] + + try: + with open(client_path) as client_config_file: + client_config_data = yaml.safe_load(client_config_file.read()) + + if ca_data is None: + with open(ca_path) as ca_file: + ca_data = base64.standard_b64encode(ca_file.read()) + + changes = [] + # Naively update the CA information for each cluster in the + # kubeconfig. + for cluster in client_config_data['clusters']: + if cluster['cluster']['certificate-authority-data'] != ca_data: + cluster['cluster']['certificate-authority-data'] = ca_data + changes.append(cluster['name']) + + if not module.check_mode: + if len(changes) > 0 and backup: + module.backup_local(client_path) + + with open(client_path, 'w') as client_config_file: + client_config_string = yaml.dump(client_config_data, default_flow_style=False) + client_config_string = client_config_string.replace('\'\'', '""') + client_config_file.write(client_config_string) + + return module.exit_json(changed=(len(changes) > 0)) + + # ignore broad-except error to avoid stack trace to ansible user + # pylint: disable=broad-except + except Exception as error: + return module.fail_json(msg=str(error)) + + +if __name__ == '__main__': + main() diff --git a/openshift-ansible.spec b/openshift-ansible.spec index 9faf3e78e..8cec36bb7 100644 --- a/openshift-ansible.spec +++ b/openshift-ansible.spec @@ -20,6 +20,7 @@ Requires: tar Requires: openshift-ansible-docs = %{version}-%{release} Requires: java-1.8.0-openjdk-headless Requires: httpd-tools +Requires: python-ruamel-yaml %description Openshift and Atomic Enterprise Ansible diff --git a/playbooks/byo/openshift-cluster/config.yml b/playbooks/byo/openshift-cluster/config.yml index d953b8ed3..86eff4ca4 100644 --- a/playbooks/byo/openshift-cluster/config.yml +++ b/playbooks/byo/openshift-cluster/config.yml @@ -1,27 +1,7 @@ --- -- name: Create initial host groups for localhost - hosts: localhost - connection: local - become: no - gather_facts: no +- include: ../../common/openshift-cluster/std_include.yml tags: - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml - - name: Evaluate group l_oo_all_hosts - add_host: - name: "{{ item }}" - groups: l_oo_all_hosts - with_items: "{{ g_all_hosts | default([]) }}" - changed_when: no - -- name: Create initial host groups for all hosts - hosts: l_oo_all_hosts - gather_facts: no - tags: - - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml - include: ../../common/openshift-cluster/config.yml vars: diff --git a/playbooks/byo/openshift-cluster/redeploy-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-certificates.yml index 753248855..ad24b9ad0 100644 --- a/playbooks/byo/openshift-cluster/redeploy-certificates.yml +++ b/playbooks/byo/openshift-cluster/redeploy-certificates.yml @@ -1,28 +1,20 @@ --- -- name: Create initial host groups for localhost - hosts: localhost - connection: local - become: no - gather_facts: no +- include: ../../common/openshift-cluster/std_include.yml tags: - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml - - name: Evaluate group l_oo_all_hosts - add_host: - name: "{{ item }}" - groups: l_oo_all_hosts - with_items: "{{ g_all_hosts | default([]) }}" - changed_when: False - -- name: Create initial host groups for all hosts - hosts: l_oo_all_hosts - gather_facts: no - tags: - - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml -- include: ../../common/openshift-cluster/redeploy-certificates.yml - vars: - openshift_deployment_type: "{{ deployment_type }}" +- include: ../../common/openshift-cluster/redeploy-certificates/etcd.yml + +- include: ../../common/openshift-cluster/redeploy-certificates/masters.yml + +- include: ../../common/openshift-cluster/redeploy-certificates/nodes.yml + +- include: ../../common/openshift-etcd/restart.yml + +- include: ../../common/openshift-master/restart.yml + +- include: ../../common/openshift-node/restart.yml + +- include: ../../common/openshift-cluster/redeploy-certificates/router.yml + +- include: ../../common/openshift-cluster/redeploy-certificates/registry.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml new file mode 100644 index 000000000..ee49364fa --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml @@ -0,0 +1,10 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/etcd.yml + +- include: ../../common/openshift-etcd/restart.yml + +- include: ../../common/openshift-master/restart.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-master-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-master-certificates.yml new file mode 100644 index 000000000..9c8248c4e --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-master-certificates.yml @@ -0,0 +1,8 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/masters.yml + +- include: ../../common/openshift-master/restart.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-node-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-node-certificates.yml new file mode 100644 index 000000000..1695111d0 --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-node-certificates.yml @@ -0,0 +1,8 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/nodes.yml + +- include: ../../common/openshift-node/restart.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-openshift-ca.yml b/playbooks/byo/openshift-cluster/redeploy-openshift-ca.yml new file mode 100644 index 000000000..e44e95467 --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-openshift-ca.yml @@ -0,0 +1,6 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/ca.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-registry-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-registry-certificates.yml new file mode 100644 index 000000000..53ee68db9 --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-registry-certificates.yml @@ -0,0 +1,6 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/registry.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-router-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-router-certificates.yml new file mode 100644 index 000000000..f8c267569 --- /dev/null +++ b/playbooks/byo/openshift-cluster/redeploy-router-certificates.yml @@ -0,0 +1,6 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-cluster/redeploy-certificates/router.yml diff --git a/playbooks/byo/openshift-etcd/filter_plugins b/playbooks/byo/openshift-etcd/filter_plugins new file mode 120000 index 000000000..99a95e4ca --- /dev/null +++ b/playbooks/byo/openshift-etcd/filter_plugins @@ -0,0 +1 @@ +../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/byo/openshift-etcd/lookup_plugins b/playbooks/byo/openshift-etcd/lookup_plugins new file mode 120000 index 000000000..ac79701db --- /dev/null +++ b/playbooks/byo/openshift-etcd/lookup_plugins @@ -0,0 +1 @@ +../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/byo/openshift-etcd/restart.yml b/playbooks/byo/openshift-etcd/restart.yml new file mode 100644 index 000000000..6713f07e3 --- /dev/null +++ b/playbooks/byo/openshift-etcd/restart.yml @@ -0,0 +1,8 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-etcd/restart.yml + vars: + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/byo/openshift-etcd/roles b/playbooks/byo/openshift-etcd/roles new file mode 120000 index 000000000..20c4c58cf --- /dev/null +++ b/playbooks/byo/openshift-etcd/roles @@ -0,0 +1 @@ +../../../roles
\ No newline at end of file diff --git a/playbooks/byo/openshift-master/restart.yml b/playbooks/byo/openshift-master/restart.yml index 3e58ccbcc..2d20f69f4 100644 --- a/playbooks/byo/openshift-master/restart.yml +++ b/playbooks/byo/openshift-master/restart.yml @@ -1,38 +1,8 @@ --- -- name: Create initial host groups for localhost - hosts: localhost - connection: local - become: no - gather_facts: no +- include: ../../common/openshift-cluster/std_include.yml tags: - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml - - name: Evaluate group l_oo_all_hosts - add_host: - name: "{{ item }}" - groups: l_oo_all_hosts - with_items: "{{ g_all_hosts | default([]) }}" - changed_when: False -- name: Create initial host groups for all hosts - hosts: l_oo_all_hosts - gather_facts: no - tags: - - always - tasks: - - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml - -- include: ../../common/openshift-cluster/evaluate_groups.yml -- include: ../../common/openshift-master/validate_restart.yml - -- name: Restart masters - hosts: oo_masters_to_config +- include: ../../common/openshift-master/restart.yml vars: - openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" - serial: 1 - tasks: - - include: restart_hosts.yml - when: openshift.common.rolling_restart_mode == 'system' - - include: restart_services.yml - when: openshift.common.rolling_restart_mode == 'services' + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/byo/openshift-node/restart.yml b/playbooks/byo/openshift-node/restart.yml new file mode 100644 index 000000000..3985a83bb --- /dev/null +++ b/playbooks/byo/openshift-node/restart.yml @@ -0,0 +1,8 @@ +--- +- include: ../../common/openshift-cluster/std_include.yml + tags: + - always + +- include: ../../common/openshift-node/restart.yml + vars: + openshift_deployment_type: "{{ deployment_type }}" diff --git a/playbooks/byo/openshift_facts.yml b/playbooks/byo/openshift_facts.yml index 025983662..fcf402fa0 100644 --- a/playbooks/byo/openshift_facts.yml +++ b/playbooks/byo/openshift_facts.yml @@ -1,29 +1,7 @@ --- -- name: Create initial host groups for localhost - hosts: localhost - connection: local - become: no - gather_facts: no +- include: ../../common/openshift-cluster/std_include.yml tags: - always - tasks: - - include_vars: ../byo/openshift-cluster/cluster_hosts.yml - - name: Evaluate group l_oo_all_hosts - add_host: - name: "{{ item }}" - groups: l_oo_all_hosts - with_items: "{{ g_all_hosts | default([]) }}" - changed_when: False - -- name: Create initial host groups for all hosts - hosts: l_oo_all_hosts - gather_facts: no - tags: - - always - tasks: - - include_vars: ../byo/openshift-cluster/cluster_hosts.yml - -- include: ../common/openshift-cluster/evaluate_groups.yml - name: Gather Cluster facts hosts: OSEv3 diff --git a/playbooks/byo/rhel_subscribe.yml b/playbooks/byo/rhel_subscribe.yml index 8e7568e33..62f62680e 100644 --- a/playbooks/byo/rhel_subscribe.yml +++ b/playbooks/byo/rhel_subscribe.yml @@ -1,29 +1,7 @@ --- -- name: Create initial host groups for localhost - hosts: localhost - connection: local - become: no - gather_facts: no +- include: ../../common/openshift-cluster/std_include.yml tags: - always - tasks: - - include_vars: openshift-cluster/cluster_hosts.yml - - name: Evaluate group l_oo_all_hosts - add_host: - name: "{{ item }}" - groups: l_oo_all_hosts - with_items: "{{ g_all_hosts | default([]) }}" - changed_when: False - -- name: Create initial host groups for all hosts - hosts: l_oo_all_hosts - gather_facts: no - tags: - - always - tasks: - - include_vars: ../byo/openshift-cluster/cluster_hosts.yml - -- include: ../common/openshift-cluster/evaluate_groups.yml - name: Subscribe hosts, update repos and update OS packages hosts: l_oo_all_hosts diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml index a95cb68b7..a0ba735ab 100644 --- a/playbooks/common/openshift-cluster/config.yml +++ b/playbooks/common/openshift-cluster/config.yml @@ -1,20 +1,4 @@ --- -- include: evaluate_groups.yml - tags: - - always - -- include: initialize_facts.yml - tags: - - always - -- include: validate_hostnames.yml - tags: - - node - -- include: initialize_openshift_version.yml - tags: - - always - - name: Set oo_option facts hosts: oo_all_hosts tags: diff --git a/playbooks/common/openshift-cluster/openshift_hosted.yml b/playbooks/common/openshift-cluster/openshift_hosted.yml index 34f1a979b..3c4a99887 100644 --- a/playbooks/common/openshift-cluster/openshift_hosted.yml +++ b/playbooks/common/openshift-cluster/openshift_hosted.yml @@ -47,12 +47,16 @@ when: ( openshift.common.version_gte_3_3_or_1_3 | bool ) and ( openshift_hosted_manage_registry | default(true) | bool ) and not (openshift.docker.hosted_registry_insecure | default(false) | bool) - name: Update master-config for publicLoggingURL - hosts: masters:!oo_first_master + hosts: oo_masters_to_config:!oo_first_master + tags: + - hosted pre_tasks: - set_fact: logging_hostname: "{{ openshift_hosted_logging_hostname | default('kibana.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true))) }}" tasks: - - include_role: - name: openshift_hosted_logging - tasks_from: update_master_config + + - block: + - include_role: + name: openshift_hosted_logging + tasks_from: update_master_config when: openshift_hosted_logging_deploy | default(false) | bool diff --git a/playbooks/common/openshift-cluster/openshift_logging.yml b/playbooks/common/openshift-cluster/openshift_logging.yml index 82f18f5e1..d96a78c4c 100644 --- a/playbooks/common/openshift-cluster/openshift_logging.yml +++ b/playbooks/common/openshift-cluster/openshift_logging.yml @@ -7,7 +7,8 @@ - name: Update Master configs hosts: masters:!oo_first_master tasks: - - include_role: - name: openshift_logging - tasks_from: update_master_config + - block: + - include_role: + name: openshift_logging + tasks_from: update_master_config when: openshift_logging_install_logging | default(false) | bool diff --git a/playbooks/common/openshift-cluster/redeploy-certificates.yml b/playbooks/common/openshift-cluster/redeploy-certificates.yml deleted file mode 100644 index a0e3f1d8a..000000000 --- a/playbooks/common/openshift-cluster/redeploy-certificates.yml +++ /dev/null @@ -1,250 +0,0 @@ ---- -- include: evaluate_groups.yml - -- include: initialize_facts.yml - -- include: initialize_openshift_version.yml - -- name: Load openshift_facts - hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config - roles: - - openshift_facts - -- name: Redeploy etcd certificates - hosts: oo_etcd_to_config - any_errors_fatal: true - vars: - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_conf_dir: /etc/etcd - etcd_generated_certs_dir: "{{ etcd_conf_dir }}/generated_certs" - - pre_tasks: - - stat: - path: "{{ etcd_generated_certs_dir }}" - register: etcd_generated_certs_dir_stat - - name: Backup etcd certificates - command: > - tar -czvf /etc/etcd/etcd-certificate-backup-{{ ansible_date_time.epoch }}.tgz - {{ etcd_conf_dir }}/ca.crt - {{ etcd_conf_dir }}/ca - {{ etcd_generated_certs_dir }} - when: etcd_generated_certs_dir_stat.stat.exists - delegate_to: "{{ etcd_ca_host }}" - run_once: true - - name: Remove existing etcd certificates - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ etcd_conf_dir }}/ca.crt" - - "{{ etcd_conf_dir }}/ca" - - "{{ etcd_generated_certs_dir }}" - roles: - - role: openshift_etcd_server_certificates - etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_certificates_redeploy: true - -- name: Redeploy master certificates - hosts: oo_masters_to_config - any_errors_fatal: true - vars: - openshift_ca_host: "{{ groups.oo_first_master.0 }}" - openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}" - pre_tasks: - # set_fact task copied from playbooks/common/openshift-master/config.yml - # so that openshift_master_default_subdomain has a default value of "" - # (emptry string). openshift_master_default_subdomain must have a default - # value for openshift_master_facts to set metrics_public_url. - # TODO: clean this up. - - set_fact: - openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}" - when: openshift_master_default_subdomain is not defined - - stat: - path: "{{ openshift_generated_configs_dir }}" - register: openshift_generated_configs_dir_stat - - name: Backup generated certificate and config directories - command: > - tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz - {{ openshift_generated_configs_dir }} - {{ openshift.common.config_base }}/master - when: openshift_generated_configs_dir_stat.stat.exists - delegate_to: "{{ openshift_ca_host }}" - run_once: true - - name: Remove generated certificate directories - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ openshift_generated_configs_dir }}" - - name: Remove generated certificates - file: - path: "{{ openshift.common.config_base }}/master/{{ item }}" - state: absent - with_items: - - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false) }}" - - "etcd.server.crt" - - "etcd.server.key" - - "master.etcd-client.crt" - - "master.etcd-client.key" - - "master.server.crt" - - "master.server.key" - - "openshift-master.crt" - - "openshift-master.key" - - "openshift-master.kubeconfig" - - name: Remove CA certificate - file: - path: "{{ openshift.common.config_base }}/master/{{ item }}" - state: absent - when: openshift_certificates_redeploy_ca | default(false) | bool - with_items: - - "ca.crt" - - "ca.key" - - "ca.serial.txt" - - "ca-bundle.crt" - roles: - - role: openshift_master_certificates - openshift_master_etcd_hosts: "{{ hostvars - | oo_select_keys(groups['oo_etcd_to_config'] | default([])) - | oo_collect('openshift.common.hostname') - | default(none, true) }}" - openshift_certificates_redeploy: true - - role: openshift_etcd_client_certificates - etcd_certificates_redeploy: true - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" - etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" - etcd_cert_prefix: "master.etcd-" - when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config - -- name: Redeploy node certificates - hosts: oo_nodes_to_config - any_errors_fatal: true - pre_tasks: - - name: Remove CA certificate - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ openshift.common.config_base }}/node/ca.crt" - roles: - - role: openshift_node_certificates - openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" - openshift_ca_host: "{{ groups.oo_first_master.0 }}" - openshift_certificates_redeploy: true - -- name: Restart etcd - hosts: oo_etcd_to_config - tasks: - - name: restart etcd - service: - name: "{{ 'etcd' if not openshift.common.is_containerized | bool else 'etcd_container' }}" - state: restarted - -- name: Stop master services - hosts: oo_masters_to_config - vars: - openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" - tasks: - - name: stop master - service: name={{ openshift.common.service_type }}-master state=stopped - when: not openshift_master_ha | bool - - name: stop master api - service: name={{ openshift.common.service_type }}-master-api state=stopped - when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' - - name: stop master controllers - service: name={{ openshift.common.service_type }}-master-controllers state=stopped - when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' - -- name: Start master services - hosts: oo_masters_to_config - serial: 1 - vars: - openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" - tasks: - - name: start master - service: name={{ openshift.common.service_type }}-master state=started - when: not openshift_master_ha | bool - - name: start master api - service: name={{ openshift.common.service_type }}-master-api state=started - when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' - - name: start master controllers - service: name={{ openshift.common.service_type }}-master-controllers state=started - when: openshift_master_ha | bool and openshift_master_cluster_method == 'native' - -- name: Restart masters (pacemaker) - hosts: oo_first_master - vars: - openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" - tasks: - - name: restart master - command: pcs resource restart master - when: openshift_master_ha | bool and openshift_master_cluster_method == 'pacemaker' - -- name: Restart nodes - hosts: oo_nodes_to_config - tasks: - - name: restart node - service: name={{ openshift.common.service_type }}-node state=restarted - -- name: Copy admin client config(s) - hosts: oo_first_master - tasks: - - name: Create temp directory for kubeconfig - command: mktemp -d /tmp/openshift-ansible-XXXXXX - register: mktemp - changed_when: False - - - name: Copy admin client config(s) - command: > - cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig - changed_when: False - -- name: Serially drain all nodes to trigger redeployments - hosts: oo_nodes_to_config - serial: 1 - any_errors_fatal: true - tasks: - - name: Determine if node is currently scheduleable - command: > - {{ openshift.common.client_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig - get node {{ openshift.node.nodename }} -o json - register: node_output - when: openshift_certificates_redeploy_ca | default(false) | bool - delegate_to: "{{ groups.oo_first_master.0 }}" - changed_when: false - - - set_fact: - was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" - when: openshift_certificates_redeploy_ca | default(false) | bool - - - name: Prepare for node draining - command: > - {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig - manage-node {{ openshift.node.nodename }} - --schedulable=false - delegate_to: "{{ groups.oo_first_master.0 }}" - when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool - - - name: Drain node - command: > - {{ openshift.common.admin_binary }} --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig - drain {{ openshift.node.nodename }} --force --delete-local-data - delegate_to: "{{ groups.oo_first_master.0 }}" - when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool - - - name: Set node schedulability - command: > - {{ openshift.common.client_binary }} adm --config={{ hostvars[groups.oo_first_master.0].mktemp.stdout }}/admin.kubeconfig - manage-node {{ openshift.node.nodename }} --schedulable=true - delegate_to: "{{ groups.oo_first_master.0 }}" - when: openshift_certificates_redeploy_ca | default(false) | bool and was_schedulable | bool - -- name: Delete temporary directory - hosts: oo_first_master - tasks: - - name: Delete temp directory - file: - name: "{{ mktemp.stdout }}" - state: absent - changed_when: False diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml new file mode 100644 index 000000000..0b1c39ba4 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml @@ -0,0 +1,353 @@ +--- +- name: Verify OpenShift version is greater than or equal to 1.2 or 3.2 + hosts: oo_first_master + tasks: + - fail: + msg: "The current OpenShift version is less than 1.2/3.2 and does not support CA bundles." + when: not openshift.common.version_gte_3_2_or_1_2 | bool + +- name: Backup existing etcd CA certificate directories + hosts: oo_etcd_to_config + roles: + - etcd_common + tasks: + - name: Determine if CA certificate directory exists + stat: + path: "{{ etcd_ca_dir }}" + register: etcd_ca_certs_dir_stat + - name: Backup generated etcd certificates + command: > + tar -czf {{ etcd_conf_dir }}/etcd-ca-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_ca_dir }} + args: + warn: no + when: etcd_ca_certs_dir_stat.stat.exists | bool + - name: Remove CA certificate directory + file: + path: "{{ etcd_ca_dir }}" + state: absent + when: etcd_ca_certs_dir_stat.stat.exists | bool + +- name: Generate new etcd CA + hosts: oo_first_etcd + roles: + - role: etcd_ca + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + +- name: Create temp directory for syncing certs + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_etcd_mktemp + changed_when: false + +- name: Distribute etcd CA to etcd hosts + hosts: oo_etcd_to_config + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + roles: + - etcd_common + tasks: + - name: Create a tarball of the etcd ca certs + command: > + tar -czvf {{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz + -C {{ etcd_ca_dir }} . + args: + creates: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz" + warn: no + delegate_to: "{{ etcd_ca_host }}" + run_once: true + - name: Retrieve etcd ca cert tarball + fetch: + src: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz" + dest: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + delegate_to: "{{ etcd_ca_host }}" + run_once: true + - name: Ensure ca directory exists + file: + path: "{{ etcd_ca_dir }}" + state: directory + - name: Unarchive etcd ca cert tarballs + unarchive: + src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/{{ etcd_ca_name }}.tgz" + dest: "{{ etcd_ca_dir }}" + - name: Read current etcd CA + slurp: + src: "{{ etcd_conf_dir }}/ca.crt" + register: g_current_etcd_ca_output + - name: Read new etcd CA + slurp: + src: "{{ etcd_ca_dir }}/ca.crt" + register: g_new_etcd_ca_output + - copy: + content: "{{ (g_new_etcd_ca_output.content|b64decode) + (g_current_etcd_ca_output.content|b64decode) }}" + dest: "{{ item }}/ca.crt" + with_items: + - "{{ etcd_conf_dir }}" + - "{{ etcd_ca_dir }}" + +- name: Retrieve etcd CA certificate + hosts: oo_first_etcd + roles: + - etcd_common + tasks: + - name: Retrieve etcd CA certificate + fetch: + src: "{{ etcd_conf_dir }}/ca.crt" + dest: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + +- name: Distribute etcd CA to masters + hosts: oo_masters_to_config + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + tasks: + - name: Deploy CA certificate, key, bundle and serial + copy: + src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/ca.crt" + dest: "{{ openshift.common.config_base }}/master/master.etcd-ca.crt" + when: groups.oo_etcd_to_config | default([]) | length > 0 + +- name: Delete temporary directory on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: + name: "{{ g_etcd_mktemp.stdout }}" + state: absent + changed_when: false + +- include: ../../../common/openshift-etcd/restart.yml + +# Update master config when ca-bundle not referenced. Services will be +# restarted below after new CA certificate has been distributed. +- name: Ensure ca-bundle.crt is referenced in master configuration + hosts: oo_masters_to_config + tasks: + - slurp: + src: "{{ openshift.common.config_base }}/master/master-config.yaml" + register: g_master_config_output + - modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: kubeletClientInfo.ca + yaml_value: ca-bundle.crt + when: (g_master_config_output.content|b64decode|from_yaml).kubeletClientInfo.ca != 'ca-bundle.crt' + - modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: serviceAccountConfig.masterCA + yaml_value: ca-bundle.crt + when: (g_master_config_output.content|b64decode|from_yaml).serviceAccountConfig.masterCA != 'ca-bundle.crt' + - modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: oauthConfig.masterCA + yaml_value: ca-bundle.crt + when: (g_master_config_output.content|b64decode|from_yaml).oauthConfig.masterCA != 'ca-bundle.crt' + - modify_yaml: + dest: "{{ openshift.common.config_base }}/master/master-config.yaml" + yaml_key: servingInfo.clientCA + yaml_value: ca-bundle.crt + when: (g_master_config_output.content|b64decode|from_yaml).servingInfo.clientCA != 'ca-bundle.crt' + +- name: Copy current OpenShift CA to legacy directory + hosts: oo_masters_to_config + pre_tasks: + - name: Create legacy-ca directory + file: + path: "{{ openshift.common.config_base }}/master/legacy-ca" + state: directory + mode: 0700 + owner: root + group: root + - command: mktemp -u XXXXXX + register: g_legacy_ca_mktemp + changed_when: false + # Copy CA certificate, key, serial and bundle to legacy-ca with a + # prefix generated by mktemp, ie. XXXXXX-ca.crt. + # + # The following roles will pick up all CA certificates matching + # /.*-ca.crt/ in the legacy-ca directory and ensure they are present + # in the OpenShift CA bundle. + # - openshift_ca + # - openshift_master_certificates + # - openshift_node_certificates + - name: Copy current OpenShift CA to legacy directory + copy: + src: "{{ openshift.common.config_base }}/master/{{ item }}" + dest: "{{ openshift.common.config_base }}/master/legacy-ca/{{ g_legacy_ca_mktemp.stdout }}-{{ item }}" + remote_src: true + # It is possible that redeploying failed and files may be missing. + # Ignore errors in this case. Files should have been copied to + # legacy-ca directory in previous run. + ignore_errors: true + with_items: + - "ca.crt" + - "ca.key" + - "ca.serial.txt" + - "ca-bundle.crt" + +- name: Generate new OpenShift CA certificate + hosts: oo_first_master + pre_tasks: + - name: Create temporary directory for creating new CA certificate + command: > + mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_new_openshift_ca_mktemp + changed_when: false + roles: + - role: openshift_ca + # Set openshift_ca_config_dir to a temporary directory where CA + # will be created. We'll replace the existing CA with the CA + # created in the temporary directory. + openshift_ca_config_dir: "{{ g_new_openshift_ca_mktemp.stdout }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_hostnames: "{{ hostvars + | oo_select_keys(groups['oo_masters_to_config'] | default([])) + | oo_collect('openshift.common.all_hostnames') + | oo_flatten | unique }}" + +- name: Create temp directory for syncing certs + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_master_mktemp + changed_when: false + +- name: Retrieve OpenShift CA + hosts: oo_first_master + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + tasks: + - name: Retrieve CA certificate, key, bundle and serial + fetch: + src: "{{ hostvars[openshift_ca_host].g_new_openshift_ca_mktemp.stdout }}/{{ item }}" + dest: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + with_items: + - ca.crt + - ca.key + - ca-bundle.crt + - ca.serial.txt + delegate_to: "{{ openshift_ca_host }}" + run_once: true + changed_when: false + +- name: Distribute OpenShift CA to masters + hosts: oo_masters_to_config + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + tasks: + - name: Deploy CA certificate, key, bundle and serial + copy: + src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/{{ item }}" + dest: "{{ openshift.common.config_base }}/master/" + with_items: + - ca.crt + - ca.key + - ca-bundle.crt + - ca.serial.txt + - name: Update master client kubeconfig CA data + kubeclient_ca: + client_path: "{{ openshift.common.config_base }}/master/openshift-master.kubeconfig" + ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt" + - name: Update admin client kubeconfig CA data + kubeclient_ca: + client_path: "{{ openshift.common.config_base }}/master/admin.kubeconfig" + ca_path: "{{ openshift.common.config_base }}/master/ca-bundle.crt" + - name: Lookup default group for ansible_ssh_user + command: "/usr/bin/id -g {{ ansible_ssh_user }}" + changed_when: false + register: _ansible_ssh_user_gid + - set_fact: + client_users: "{{ [ansible_ssh_user, 'root'] | unique }}" + - name: Create the client config dir(s) + file: + path: "~{{ item }}/.kube" + state: directory + mode: 0700 + owner: "{{ item }}" + group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout }}" + with_items: "{{ client_users }}" + - name: Copy the admin client config(s) + copy: + src: "{{ openshift.common.config_base }}/master/admin.kubeconfig" + dest: "~{{ item }}/.kube/config" + remote_src: yes + with_items: "{{ client_users }}" + - name: Update the permissions on the admin client config(s) + file: + path: "~{{ item }}/.kube/config" + state: file + mode: 0700 + owner: "{{ item }}" + group: "{{ 'root' if item == 'root' else _ansible_ssh_user_gid.stdout }}" + with_items: "{{ client_users }}" + +- include: ../../../common/openshift-master/restart.yml + +- name: Distribute OpenShift CA certificate to nodes + hosts: oo_nodes_to_config + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + tasks: + - copy: + src: "{{ hostvars['localhost'].g_master_mktemp.stdout }}/ca-bundle.crt" + dest: "{{ openshift.common.config_base }}/node/ca.crt" + - name: Copy OpenShift CA to system CA trust + copy: + src: "{{ item.cert }}" + dest: "/etc/pki/ca-trust/source/anchors/{{ item.id }}-{{ item.cert | basename }}" + remote_src: yes + with_items: + - id: openshift + cert: "{{ openshift.common.config_base }}/node/ca.crt" + notify: + - update ca trust + - name: Update node client kubeconfig CA data + kubeclient_ca: + client_path: "{{ openshift.common.config_base }}/node/system:node:{{ openshift.common.hostname }}.kubeconfig" + ca_path: "{{ openshift.common.config_base }}/node/ca.crt" + handlers: + # Normally this handler would restart docker after updating ca + # trust. We'll do that when we restart nodes to avoid restarting + # docker on all nodes in parallel. + - name: update ca trust + command: update-ca-trust + +- name: Delete temporary directory on CA host + hosts: oo_first_master + tasks: + - file: + path: "{{ g_new_openshift_ca_mktemp.stdout }}" + state: absent + +- name: Delete temporary directory on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: + name: "{{ g_master_mktemp.stdout }}" + state: absent + changed_when: false + +- include: ../../../common/openshift-node/restart.yml diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml new file mode 100644 index 000000000..2963a5940 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml @@ -0,0 +1,66 @@ +--- +- name: Backup and remove generated etcd certificates + hosts: oo_first_etcd + any_errors_fatal: true + roles: + - etcd_common + post_tasks: + - name: Determine if generated etcd certificates exist + stat: + path: "{{ etcd_conf_dir }}/generated_certs" + register: etcd_generated_certs_dir_stat + - name: Backup generated etcd certificates + command: > + tar -czf {{ etcd_conf_dir }}/etcd-generated-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_conf_dir }}/generated_certs + args: + warn: no + when: etcd_generated_certs_dir_stat.stat.exists | bool + - name: Remove generated etcd certificates + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ etcd_conf_dir }}/generated_certs" + +- name: Backup and removed deployed etcd certificates + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - etcd_common + post_tasks: + - name: Backup etcd certificates + command: > + tar -czvf /etc/etcd/etcd-server-certificate-backup-{{ ansible_date_time.epoch }}.tgz + {{ etcd_conf_dir }}/ca.crt + {{ etcd_conf_dir }}/server.crt + {{ etcd_conf_dir }}/server.key + {{ etcd_conf_dir }}/peer.crt + {{ etcd_conf_dir }}/peer.key + args: + warn: no + +- name: Redeploy etcd certificates + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_server_certificates + etcd_certificates_redeploy: true + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + +- name: Redeploy etcd client certificates for masters + hosts: oo_masters_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_client_certificates + etcd_certificates_redeploy: true + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" + etcd_cert_prefix: "master.etcd-" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}" + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins b/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins new file mode 120000 index 000000000..b1213dedb --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/filter_plugins @@ -0,0 +1 @@ +../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/library b/playbooks/common/openshift-cluster/redeploy-certificates/library new file mode 120000 index 000000000..9a53f009d --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/library @@ -0,0 +1 @@ +../../../../library
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins b/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins new file mode 120000 index 000000000..aff753026 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/lookup_plugins @@ -0,0 +1 @@ +../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml b/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml new file mode 100644 index 000000000..f653a111f --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/masters.yml @@ -0,0 +1,45 @@ +--- +- name: Redeploy master certificates + hosts: oo_masters_to_config + any_errors_fatal: true + vars: + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}" + pre_tasks: + - stat: + path: "{{ openshift_generated_configs_dir }}" + register: openshift_generated_configs_dir_stat + - name: Backup generated certificate and config directories + command: > + tar -czvf /etc/origin/master-node-cert-config-backup-{{ ansible_date_time.epoch }}.tgz + {{ openshift_generated_configs_dir }} + {{ openshift.common.config_base }}/master + when: openshift_generated_configs_dir_stat.stat.exists + delegate_to: "{{ openshift_ca_host }}" + run_once: true + - name: Remove generated certificate directories + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ openshift_generated_configs_dir }}" + - name: Remove generated certificates + file: + path: "{{ openshift.common.config_base }}/master/{{ item }}" + state: absent + with_items: + - "{{ hostvars[inventory_hostname] | certificates_to_synchronize(include_keys=false, include_ca=false) }}" + - "etcd.server.crt" + - "etcd.server.key" + - "master.server.crt" + - "master.server.key" + - "openshift-master.crt" + - "openshift-master.key" + - "openshift-master.kubeconfig" + roles: + - role: openshift_master_certificates + openshift_master_etcd_hosts: "{{ hostvars + | oo_select_keys(groups['oo_etcd_to_config'] | default([])) + | oo_collect('openshift.common.hostname') + | default(none, true) }}" + openshift_certificates_redeploy: true diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml b/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml new file mode 100644 index 000000000..4990a03f2 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/nodes.yml @@ -0,0 +1,29 @@ +--- +- name: Ensure node directory is absent from generated configs + hosts: oo_first_master + tasks: + # The generated configs directory (/etc/origin/generated-configs) is + # backed up during redeployment of the control plane certificates. + # We need to ensure that the generated config directory for + # individual nodes has been deleted before continuing, so verify + # that it is missing here. + - name: Ensure node directories and tarballs are absent from generated configs + shell: > + rm -rf {{ openshift.common.config_base }}/generated-configs/node-* + args: + warn: no + +- name: Redeploy node certificates + hosts: oo_nodes_to_config + pre_tasks: + - name: Remove CA certificate + file: + path: "{{ item }}" + state: absent + with_items: + - "{{ openshift.common.config_base }}/node/ca.crt" + roles: + - role: openshift_node_certificates + openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_certificates_redeploy: true diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml b/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml new file mode 100644 index 000000000..18b93e1d6 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/registry.yml @@ -0,0 +1,93 @@ +--- +- name: Update registry certificates + hosts: oo_first_master + vars: + tasks: + - name: Create temp directory for kubeconfig + command: mktemp -d /tmp/openshift-ansible-XXXXXX + register: mktemp + changed_when: false + + - name: Copy admin client config(s) + command: > + cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig + changed_when: false + + - name: Determine if docker-registry exists + command: > + {{ openshift.common.client_binary }} get dc/docker-registry -o json + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + register: l_docker_registry_dc + failed_when: false + changed_when: false + + - set_fact: + docker_registry_env_vars: "{{ ((l_docker_registry_dc.stdout | from_json)['spec']['template']['spec']['containers'][0]['env'] + | oo_collect('name')) + | default([]) }}" + docker_registry_secrets: "{{ ((l_docker_registry_dc.stdout | from_json)['spec']['template']['spec']['volumes'] + | oo_collect('secret') + | oo_collect('secretName')) + | default([]) }}" + changed_when: false + when: l_docker_registry_dc.rc == 0 + + # Replace dc/docker-registry environment variable certificate data if set. + - name: Update docker-registry environment variables + shell: > + {{ openshift.common.client_binary }} env dc/docker-registry + OPENSHIFT_CA_DATA="$(cat /etc/origin/master/ca.crt)" + OPENSHIFT_CERT_DATA="$(cat /etc/origin/master/openshift-registry.crt)" + OPENSHIFT_KEY_DATA="$(cat /etc/origin/master/openshift-registry.key)" + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + when: l_docker_registry_dc.rc == 0 and 'OPENSHIFT_CA_DATA' in docker_registry_env_vars and 'OPENSHIFT_CERT_DATA' in docker_registry_env_vars and 'OPENSHIFT_KEY_DATA' in docker_registry_env_vars + + # Replace dc/docker-registry certificate secret contents if set. + - block: + - name: Retrieve registry service IP + command: > + {{ openshift.common.client_binary }} get service docker-registry + -o jsonpath='{.spec.clusterIP}' + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + register: docker_registry_service_ip + changed_when: false + + - set_fact: + docker_registry_route_hostname: "{{ 'docker-registry-default.' ~ (openshift.master.default_subdomain | default('router.default.svc.cluster.local', true)) }}" + changed_when: false + + - name: Generate registry certificate + command: > + {{ openshift.common.client_binary }} adm ca create-server-cert + --signer-cert={{ openshift.common.config_base }}/master/ca.crt + --signer-key={{ openshift.common.config_base }}/master/ca.key + --signer-serial={{ openshift.common.config_base }}/master/ca.serial.txt + --hostnames="{{ docker_registry_service_ip.stdout }},docker-registry.default.svc.cluster.local,{{ docker_registry_route_hostname }}" + --cert={{ openshift.common.config_base }}/master/registry.crt + --key={{ openshift.common.config_base }}/master/registry.key + + - name: Update registry certificates secret + shell: > + {{ openshift.common.client_binary }} secret new registry-certificates + {{ openshift.common.config_base }}/master/registry.crt + {{ openshift.common.config_base }}/master/registry.key + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + -o json | oc replace -f - + when: l_docker_registry_dc.rc == 0 and 'registry-certificates' in docker_registry_secrets and 'REGISTRY_HTTP_TLS_CERTIFICATE' in docker_registry_env_vars and 'REGISTRY_HTTP_TLS_KEY' in docker_registry_env_vars + + - name: Redeploy docker registry + command: > + {{ openshift.common.client_binary }} deploy dc/docker-registry + --latest + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + + - name: Delete temp directory + file: + name: "{{ mktemp.stdout }}" + state: absent + changed_when: False diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/roles b/playbooks/common/openshift-cluster/redeploy-certificates/roles new file mode 120000 index 000000000..4bdbcbad3 --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/roles @@ -0,0 +1 @@ +../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/router.yml b/playbooks/common/openshift-cluster/redeploy-certificates/router.yml new file mode 100644 index 000000000..03d64685d --- /dev/null +++ b/playbooks/common/openshift-cluster/redeploy-certificates/router.yml @@ -0,0 +1,79 @@ +--- +- name: Update router certificates + hosts: oo_first_master + vars: + tasks: + - name: Create temp directory for kubeconfig + command: mktemp -d /tmp/openshift-ansible-XXXXXX + register: mktemp + changed_when: false + + - name: Copy admin client config(s) + command: > + cp {{ openshift.common.config_base }}/master//admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig + changed_when: false + + - name: Determine if router exists + command: > + {{ openshift.common.client_binary }} get dc/router -o json + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + register: l_router_dc + failed_when: false + changed_when: false + + - set_fact: + router_env_vars: "{{ ((l_router_dc.stdout | from_json)['spec']['template']['spec']['containers'][0]['env'] + | oo_collect('name')) + | default([]) }}" + router_secrets: "{{ ((l_router_dc.stdout | from_json)['spec']['template']['spec']['volumes'] + | oo_collect('secret') + | oo_collect('secretName')) + | default([]) }}" + changed_when: false + when: l_router_dc.rc == 0 + + - name: Update router environment variables + shell: > + {{ openshift.common.client_binary }} env dc/router + OPENSHIFT_CA_DATA="$(cat /etc/origin/master/ca.crt)" + OPENSHIFT_CERT_DATA="$(cat /etc/origin/master/openshift-router.crt)" + OPENSHIFT_KEY_DATA="$(cat /etc/origin/master/openshift-router.key)" + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + when: l_router_dc.rc == 0 and 'OPENSHIFT_CA_DATA' in router_env_vars and 'OPENSHIFT_CERT_DATA' in router_env_vars and 'OPENSHIFT_KEY_DATA' in router_env_vars + + - block: + - name: Generate router certificate + command: > + {{ openshift.common.client_binary }} adm ca create-server-cert + --hostnames=router.default.svc,router.default.svc.cluster.local + --signer-cert={{ openshift.common.config_base }}/master/service-signer.crt + --signer-key={{ openshift.common.config_base }}/master/service-signer.key + --signer-serial={{ openshift.common.config_base }}/master/ca.serial.txt + --cert={{ mktemp.stdout }}/tls.crt + --key={{ mktemp.stdout }}/tls.key + + - name: Update router certificates secret + shell: > + {{ openshift.common.client_binary }} secret new router-certs + {{ mktemp.stdout }}/tls.crt + {{ mktemp.stdout }}/tls.key + --type=kubernetes.io/tls + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + -o json | oc replace -f - + when: l_router_dc.rc == 0 and 'router-certs' in router_secrets + + - name: Redeploy router + command: > + {{ openshift.common.client_binary }} deploy dc/router + --latest + --config={{ mktemp.stdout }}/admin.kubeconfig + -n default + + - name: Delete temp directory + file: + name: "{{ mktemp.stdout }}" + state: absent + changed_when: False diff --git a/playbooks/common/openshift-cluster/std_include.yml b/playbooks/common/openshift-cluster/std_include.yml new file mode 100644 index 000000000..078991b12 --- /dev/null +++ b/playbooks/common/openshift-cluster/std_include.yml @@ -0,0 +1,42 @@ +--- +- name: Create initial host groups for localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tags: + - always + tasks: + - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml + - name: Evaluate group l_oo_all_hosts + add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: "{{ g_all_hosts | default([]) }}" + changed_when: no + +- name: Create initial host groups for all hosts + hosts: l_oo_all_hosts + gather_facts: no + tags: + - always + tasks: + - include_vars: ../../byo/openshift-cluster/cluster_hosts.yml + - set_fact: + openshift_deployment_type: "{{ deployment_type }}" + +- include: evaluate_groups.yml + tags: + - always + +- include: initialize_facts.yml + tags: + - always + +- include: validate_hostnames.yml + tags: + - node + +- include: initialize_openshift_version.yml + tags: + - always diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index 9cad931af..db2c27919 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -229,3 +229,56 @@ tasks: - include: docker/upgrade.yml when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool + +- name: Drain and upgrade master nodes + hosts: oo_masters_to_config:&oo_nodes_to_upgrade + # This var must be set with -e on invocation, as it is not a per-host inventory var + # and is evaluated early. Values such as "20%" can also be used. + serial: "{{ openshift_upgrade_nodes_serial | default(1) }}" + any_errors_fatal: true + + pre_tasks: + # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node + # or docker actually needs an upgrade before proceeding. Perhaps best to save this until + # we merge upgrade functionality into the base roles and a normal config.yml playbook run. + - name: Determine if node is currently scheduleable + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.node.nodename | lower }} -o json + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + changed_when: false + + - set_fact: + was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" + + - name: Mark node unschedulable + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + # NOTE: There is a transient "object has been modified" error here, allow a couple + # retries for a more reliable upgrade. + register: node_unsched + until: node_unsched.rc == 0 + retries: 3 + delay: 1 + + - name: Drain Node for Kubelet upgrade + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data + delegate_to: "{{ groups.oo_first_master.0 }}" + + roles: + - openshift_facts + - docker + - openshift_node_upgrade + + post_tasks: + - name: Set node schedulability + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: was_schedulable | bool + register: node_sched + until: node_sched.rc == 0 + retries: 3 + delay: 1 diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml index a6a49e5ff..59188c570 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_nodes.yml @@ -1,16 +1,11 @@ --- - name: Drain and upgrade nodes - hosts: oo_nodes_to_upgrade + hosts: oo_nodes_to_upgrade:!oo_masters_to_config # This var must be set with -e on invocation, as it is not a per-host inventory var # and is evaluated early. Values such as "20%" can also be used. serial: "{{ openshift_upgrade_nodes_serial | default(1) }}" any_errors_fatal: true - roles: - - openshift_facts - - docker - handlers: - - include: ../../../../roles/openshift_node/handlers/main.yml - static: yes + pre_tasks: # TODO: To better handle re-trying failed upgrades, it would be nice to check if the node # or docker actually needs an upgrade before proceeding. Perhaps best to save this until @@ -21,17 +16,14 @@ register: node_output delegate_to: "{{ groups.oo_first_master.0 }}" changed_when: false - when: inventory_hostname in groups.oo_nodes_to_upgrade - set_fact: was_schedulable: "{{ 'unschedulable' not in (node_output.stdout | from_json).spec }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade - - name: Mark unschedulable if host is a node + - name: Mark node unschedulable command: > {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade # NOTE: There is a transient "object has been modified" error here, allow a couple # retries for a more reliable upgrade. register: node_unsched @@ -43,83 +35,18 @@ command: > {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade - - tasks: - - - include: docker/upgrade.yml - vars: - # We will restart Docker ourselves after everything is ready: - skip_docker_restart: True - when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool - - - include: "{{ node_config_hook }}" - when: node_config_hook is defined and inventory_hostname in groups.oo_nodes_to_upgrade - - - include: rpm_upgrade.yml - vars: - component: "node" - openshift_version: "{{ openshift_pkg_version | default('') }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool - - - name: Remove obsolete docker-sdn-ovs.conf - file: path=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf state=absent - when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>=')) - - - include: containerized_node_upgrade.yml - when: inventory_hostname in groups.oo_nodes_to_upgrade and openshift.common.is_containerized | bool - - - name: Ensure containerized services stopped before Docker restart - service: name={{ item }} state=stopped - with_items: - - etcd_container - - openvswitch - - "{{ openshift.common.service_type }}-master" - - "{{ openshift.common.service_type }}-master-api" - - "{{ openshift.common.service_type }}-master-controllers" - - "{{ openshift.common.service_type }}-node" - failed_when: false - when: openshift.common.is_containerized | bool - - name: Upgrade openvswitch - package: - name: openvswitch - state: latest - register: ovs_pkg - when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool - - - name: Restart openvswitch - systemd: - name: openvswitch - state: restarted - when: - - inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool - - ovs_pkg | changed - - # Mandatory Docker restart, ensure all containerized services are running: - - include: docker/restart.yml - - - name: Restart rpm node service - service: name="{{ openshift.common.service_type }}-node" state=restarted - when: inventory_hostname in groups.oo_nodes_to_upgrade and not openshift.common.is_containerized | bool - - - name: Wait for node to be ready - command: > - {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers - register: node_output - delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade - until: "{{ node_output.stdout.split()[1].startswith('Ready')}}" - # Give the node two minutes to come back online. Note that we pre-pull images now - # so containerized services should restart quickly as well. - retries: 24 - delay: 5 + roles: + - openshift_facts + - docker + - openshift_node_upgrade + post_tasks: - name: Set node schedulability command: > {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true delegate_to: "{{ groups.oo_first_master.0 }}" - when: inventory_hostname in groups.oo_nodes_to_upgrade and was_schedulable | bool + when: was_schedulable | bool register: node_sched until: node_sched.rc == 0 retries: 3 diff --git a/playbooks/common/openshift-etcd/restart.yml b/playbooks/common/openshift-etcd/restart.yml new file mode 100644 index 000000000..196c86f28 --- /dev/null +++ b/playbooks/common/openshift-etcd/restart.yml @@ -0,0 +1,9 @@ +--- +- name: Restart etcd + hosts: oo_etcd_to_config + serial: 1 + tasks: + - name: restart etcd + service: + name: "{{ 'etcd' if not openshift.common.is_containerized | bool else 'etcd_container' }}" + state: restarted diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml new file mode 100644 index 000000000..b35368bf1 --- /dev/null +++ b/playbooks/common/openshift-master/restart.yml @@ -0,0 +1,19 @@ +--- +- include: ../../common/openshift-master/validate_restart.yml + +- name: Restart masters + hosts: oo_masters_to_config + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + serial: 1 + handlers: + - include: roles/openshift_master/handlers/main.yml + static: yes + roles: + - openshift_facts + post_tasks: + - include: ../../common/openshift-master/restart_hosts.yml + when: openshift_rolling_restart_mode | default('services') == 'system' + + - include: ../../common/openshift-master/restart_services.yml + when: openshift_rolling_restart_mode | default('services') == 'services' diff --git a/playbooks/common/openshift-node/restart.yml b/playbooks/common/openshift-node/restart.yml new file mode 100644 index 000000000..6e9b1cca3 --- /dev/null +++ b/playbooks/common/openshift-node/restart.yml @@ -0,0 +1,47 @@ +--- +- name: Restart nodes + hosts: oo_nodes_to_config + serial: "{{ openshift_restart_nodes_serial | default(1) }}" + tasks: + - name: Restart docker + service: name=docker state=restarted + + - name: Update docker facts + openshift_facts: + role: docker + + - name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + + - name: Wait for master API to come back online + wait_for: + host: "{{ openshift.common.hostname }}" + state: started + delay: 10 + port: "{{ openshift.master.api_port }}" + when: inventory_hostname in groups.oo_masters_to_config + + - name: restart node + service: + name: "{{ openshift.common.service_type }}-node" + state: restarted + + - name: Wait for node to be ready + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + when: inventory_hostname in groups.oo_nodes_to_config + until: "{{ node_output.stdout.split()[1].startswith('Ready')}}" + # Give the node two minutes to come back online. + retries: 24 + delay: 5 diff --git a/roles/etcd_ca/tasks/main.yml b/roles/etcd_ca/tasks/main.yml index c4d5efa14..b4dea4a07 100644 --- a/roles/etcd_ca/tasks/main.yml +++ b/roles/etcd_ca/tasks/main.yml @@ -60,7 +60,8 @@ delegate_to: "{{ etcd_ca_host }}" run_once: true -- command: > +- name: Create etcd CA certificate + command: > openssl req -config {{ etcd_openssl_conf }} -newkey rsa:4096 -keyout {{ etcd_ca_key }} -new -out {{ etcd_ca_cert }} -x509 -extensions {{ etcd_ca_exts_self }} -batch -nodes diff --git a/roles/etcd_server_certificates/tasks/main.yml b/roles/etcd_server_certificates/tasks/main.yml index 1acdf1c85..242c1e997 100644 --- a/roles/etcd_server_certificates/tasks/main.yml +++ b/roles/etcd_server_certificates/tasks/main.yml @@ -58,6 +58,7 @@ ~ etcd_cert_prefix ~ 'server.crt' }}" environment: SAN: "IP:{{ etcd_ip }}" + when: etcd_server_certs_missing | bool delegate_to: "{{ etcd_ca_host }}" - name: Create the peer csr diff --git a/roles/lib_openshift/tasks/main.yml b/roles/lib_openshift/tasks/main.yml new file mode 100644 index 000000000..2980c8a8d --- /dev/null +++ b/roles/lib_openshift/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: lib_openshift ensure python-ruamel-yaml package is on target + package: + name: python-ruamel-yaml + state: present diff --git a/roles/lib_utils/tasks/main.yml b/roles/lib_utils/tasks/main.yml new file mode 100644 index 000000000..8a350da88 --- /dev/null +++ b/roles/lib_utils/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: lib_utils ensure python-ruamel-yaml package is on target + package: + name: python-ruamel-yaml + state: present diff --git a/roles/openshift_ca/tasks/main.yml b/roles/openshift_ca/tasks/main.yml index e21397170..4efc77f11 100644 --- a/roles/openshift_ca/tasks/main.yml +++ b/roles/openshift_ca/tasks/main.yml @@ -41,10 +41,9 @@ run_once: true - set_fact: - master_ca_missing: "{{ true if openshift_certificates_redeploy | default(false) | bool - else False in (g_master_ca_stat_result.results - | oo_collect(attribute='stat.exists') - | list) }}" + master_ca_missing: "{{ False in (g_master_ca_stat_result.results + | oo_collect(attribute='stat.exists') + | list) }}" run_once: true - name: Retain original serviceaccount keys @@ -61,7 +60,6 @@ copy: src: "{{ item.src }}" dest: "{{ openshift_ca_config_dir }}/{{ item.dest }}" - force: "{{ true if openshift_certificates_redeploy_ca | default(false) | bool else false }}" with_items: - src: "{{ (openshift_master_ca_certificate | default({'certfile':none})).certfile }}" dest: ca.crt @@ -73,25 +71,35 @@ - name: Create ca serial copy: - content: "1" + content: "00" dest: "{{ openshift_ca_config_dir }}/ca.serial.txt" - force: "{{ true if openshift_certificates_redeploy | default(false) | bool else false }}" + force: "{{ openshift_certificates_redeploy | default(false) | bool }}" when: openshift_master_ca_certificate is defined delegate_to: "{{ openshift_ca_host }}" run_once: true +- find: + paths: "{{ openshift.common.config_base }}/master/legacy-ca/" + patterns: ".*-ca.crt" + use_regex: true + register: g_master_legacy_ca_result + +# This should NOT replace the CA due to --overwrite=false when a CA already exists. - name: Create the master certificates if they do not already exist command: > {{ openshift.common.client_binary }} adm create-master-certs {% for named_ca_certificate in openshift.master.named_certificates | default([]) | oo_collect('cafile') %} --certificate-authority {{ named_ca_certificate }} {% endfor %} + {% for legacy_ca_certificate in g_master_legacy_ca_result.files | default([]) | oo_collect('path') %} + --certificate-authority {{ legacy_ca_certificate }} + {% endfor %} --hostnames={{ openshift.common.all_hostnames | join(',') }} --master={{ openshift.master.api_url }} --public-master={{ openshift.master.public_api_url }} --cert-dir={{ openshift_ca_config_dir }} --overwrite=false - when: master_ca_missing | bool + when: master_ca_missing | bool or openshift_certificates_redeploy | default(false) | bool delegate_to: "{{ openshift_ca_host }}" run_once: true diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 7c61da950..e72ab26fc 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -2027,7 +2027,7 @@ class OpenShiftFacts(object): if 'docker' in roles: docker = dict(disable_push_dockerhub=False, - options='--log-driver=json-file --log-opt max-size=50m') + options='--log-driver=journald') # NOTE: This is a workaround for a dnf output racecondition that can occur in # some situations. See https://bugzilla.redhat.com/show_bug.cgi?id=918184 if self.system_facts['ansible_pkg_mgr'] == 'dnf': diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md index 856cfa2b9..8651e06e7 100644 --- a/roles/openshift_logging/README.md +++ b/roles/openshift_logging/README.md @@ -36,6 +36,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log - `openshift_logging_curator_cpu_limit`: The amount of CPU to allocate to Curator. Default is '100m'. - `openshift_logging_curator_memory_limit`: The amount of memory to allocate to Curator. Unset if not specified. - `openshift_logging_curator_nodeselector`: A map of labels (e.g. {"node":"infra","region":"west"} to select the nodes where the curator pod will land. +- `openshift_logging_image_pull_secret`: The name of an existing pull secret to link to the logging service accounts - `openshift_logging_kibana_hostname`: The Kibana hostname. Defaults to 'kibana.example.com'. - `openshift_logging_kibana_cpu_limit`: The amount of CPU to allocate to Kibana or unset if not specified. diff --git a/roles/openshift_logging/tasks/generate_routes.yaml b/roles/openshift_logging/tasks/generate_routes.yaml index 25877ebff..60694f67e 100644 --- a/roles/openshift_logging/tasks/generate_routes.yaml +++ b/roles/openshift_logging/tasks/generate_routes.yaml @@ -17,5 +17,5 @@ - {name: logging-kibana-ops, host: "{{openshift_logging_kibana_ops_hostname}}"} loop_control: loop_var: route_info - when: (route_info.name == 'logging-kibana-ops' and openshift_logging_use_ops) or route_info.name == 'logging-kibana' + when: (route_info.name == 'logging-kibana-ops' and openshift_logging_use_ops | bool) or route_info.name == 'logging-kibana' changed_when: no diff --git a/roles/openshift_logging/tasks/generate_services.yaml b/roles/openshift_logging/tasks/generate_services.yaml index 8eaac76c4..5091c1209 100644 --- a/roles/openshift_logging/tasks/generate_services.yaml +++ b/roles/openshift_logging/tasks/generate_services.yaml @@ -52,7 +52,7 @@ selector: provider: openshift component: es-ops - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool check_mode: no changed_when: no @@ -67,7 +67,7 @@ selector: provider: openshift component: es-ops - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool check_mode: no changed_when: no @@ -82,6 +82,6 @@ selector: provider: openshift component: kibana-ops - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool check_mode: no changed_when: no diff --git a/roles/openshift_logging/tasks/install_curator.yaml b/roles/openshift_logging/tasks/install_curator.yaml index fcfce4e1e..5b474ff39 100644 --- a/roles/openshift_logging/tasks/install_curator.yaml +++ b/roles/openshift_logging/tasks/install_curator.yaml @@ -15,7 +15,7 @@ register: curator_ops_replica_count when: - not ansible_check_mode - - openshift_logging_use_ops + - openshift_logging_use_ops | bool ignore_errors: yes changed_when: no @@ -48,6 +48,6 @@ curator_memory_limit: "{{openshift_logging_curator_ops_memory_limit }}" replicas: "{{curator_ops_replica_count.stdout | default (0)}}" curator_node_selector: "{{openshift_logging_curator_ops_nodeselector | default({}) }}" - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool check_mode: no changed_when: no diff --git a/roles/openshift_logging/tasks/install_elasticsearch.yaml b/roles/openshift_logging/tasks/install_elasticsearch.yaml index 64e983557..1d6e55e44 100644 --- a/roles/openshift_logging/tasks/install_elasticsearch.yaml +++ b/roles/openshift_logging/tasks/install_elasticsearch.yaml @@ -56,7 +56,7 @@ es_dcs: "{{openshift_logging_facts.elasticsearch_ops.deploymentconfigs}}" cluster_size: "{{openshift_logging_es_ops_cluster_size}}" when: - - openshift_logging_use_ops + - openshift_logging_use_ops | bool - "{{es_dcs | length - openshift_logging_es_ops_cluster_size | abs > 1}}" check_mode: no @@ -71,7 +71,7 @@ openshift_logging_es_pvc_dynamic: "{{openshift_logging_es_ops_pvc_dynamic}}" openshift_logging_es_pv_selector: "{{openshift_logging_es_ops_pv_selector}}" when: - - openshift_logging_use_ops + - openshift_logging_use_ops | bool check_mode: no - name: Init pool of DeploymentConfig names for Elasticsearch Ops @@ -80,7 +80,7 @@ loop_control: loop_var: deploy_name when: - - openshift_logging_use_ops + - openshift_logging_use_ops | bool - name: Create new DeploymentConfig names for Elasticsearch Ops set_fact: es_ops_dc_pool={{es_ops_dc_pool | default([]) + [deploy_name]}} @@ -92,7 +92,7 @@ cluster_size: "{{openshift_logging_es_ops_cluster_size}}" with_sequence: count={{ openshift_logging_es_ops_cluster_size | int - openshift_logging_current_es_ops_size | int }} when: - - openshift_logging_use_ops + - openshift_logging_use_ops | bool check_mode: no - name: Generate Elasticsearch DeploymentConfig for Ops @@ -116,6 +116,6 @@ with_indexed_items: - "{{ es_ops_dc_pool | default([]) }}" when: - - openshift_logging_use_ops + - openshift_logging_use_ops | bool check_mode: no changed_when: no diff --git a/roles/openshift_logging/tasks/install_kibana.yaml b/roles/openshift_logging/tasks/install_kibana.yaml index f4df7de0c..3aeff2cac 100644 --- a/roles/openshift_logging/tasks/install_kibana.yaml +++ b/roles/openshift_logging/tasks/install_kibana.yaml @@ -15,7 +15,7 @@ register: kibana_ops_replica_count when: - not ansible_check_mode - - openshift_logging_use_ops + - openshift_logging_use_ops | bool ignore_errors: yes changed_when: no @@ -55,6 +55,6 @@ kibana_proxy_memory_limit: "{{openshift_logging_kibana_ops_proxy_memory_limit }}" replicas: "{{kibana_ops_replica_count.stdout | default (0)}}" kibana_node_selector: "{{openshift_logging_kibana_ops_nodeselector | default({}) }}" - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool check_mode: no changed_when: no diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml index 00c79ee5e..d52429f03 100644 --- a/roles/openshift_logging/tasks/install_logging.yaml +++ b/roles/openshift_logging/tasks/install_logging.yaml @@ -57,6 +57,28 @@ loop_var: file when: ansible_check_mode + # TODO replace task with oc_secret module that supports + # linking when available +- name: Link Pull Secrets With Service Accounts + include: oc_secret.yaml + vars: + kubeconfig: "{{ mktemp.stdout }}/admin.kubeconfig" + subcommand: link + service_account: "{{sa_account}}" + secret_name: "{{openshift_logging_image_pull_secret}}" + add_args: "--for=pull" + with_items: + - default + - aggregated-logging-elasticsearch + - aggregated-logging-kibana + - aggregated-logging-fluentd + - aggregated-logging-curator + register: link_pull_secret + loop_control: + loop_var: sa_account + when: openshift_logging_image_pull_secret is defined + failed_when: link_pull_secret.rc != 0 + - name: Scaling up cluster include: start_cluster.yaml when: start_cluster | default(true) | bool diff --git a/roles/openshift_logging/tasks/main.yaml b/roles/openshift_logging/tasks/main.yaml index 36fb827c3..4c718805e 100644 --- a/roles/openshift_logging/tasks/main.yaml +++ b/roles/openshift_logging/tasks/main.yaml @@ -12,10 +12,6 @@ - debug: msg="Created temp dir {{mktemp.stdout}}" -- name: Ensuring ruamel.yaml package is on target - command: yum install -y ruamel.yaml - check_mode: no - - name: Copy the admin client config(s) command: > cp {{ openshift_master_config_dir }}/admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig diff --git a/roles/openshift_logging/tasks/oc_apply.yaml b/roles/openshift_logging/tasks/oc_apply.yaml index c362b7fca..cb9509de1 100644 --- a/roles/openshift_logging/tasks/oc_apply.yaml +++ b/roles/openshift_logging/tasks/oc_apply.yaml @@ -1,12 +1,13 @@ --- - name: Checking generation of {{file_content.kind}} {{file_content.metadata.name}} - shell: > + command: > {{ openshift.common.client_binary }} --config={{ kubeconfig }} get {{file_content.kind}} {{file_content.metadata.name}} -o jsonpath='{.metadata.resourceVersion}' - -n {{namespace}} || echo 0 + -n {{namespace}} register: generation_init + failed_when: "'not found' not in generation_init.stderr and generation_init.stdout == ''" changed_when: no - name: Applying {{file_name}} @@ -19,11 +20,33 @@ changed_when: no - name: Determine change status of {{file_content.kind}} {{file_content.metadata.name}} - shell: > + command: > {{ openshift.common.client_binary }} --config={{ kubeconfig }} get {{file_content.kind}} {{file_content.metadata.name}} -o jsonpath='{.metadata.resourceVersion}' - -n {{namespace}} || echo 0 + -n {{namespace}} register: generation_changed - failed_when: "'error' in generation_changed.stderr" - changed_when: generation_changed.stdout | int > generation_init.stdout | int + failed_when: "'not found' not in generation_changed.stderr and generation_changed.stdout == ''" + changed_when: generation_changed.stdout | default (0) | int > generation_init.stdout | default(0) | int + when: + - "'field is immutable' not in generation_apply.stderr" + +- name: Removing previous {{file_name}} + command: > + {{ openshift.common.client_binary }} --config={{ kubeconfig }} + delete -f {{ file_name }} + -n {{ namespace }} + register: generation_delete + failed_when: "'error' in generation_delete.stderr" + changed_when: generation_delete.rc == 0 + when: "'field is immutable' in generation_apply.stderr" + +- name: Recreating {{file_name}} + command: > + {{ openshift.common.client_binary }} --config={{ kubeconfig }} + apply -f {{ file_name }} + -n {{ namespace }} + register: generation_apply + failed_when: "'error' in generation_apply.stderr" + changed_when: generation_apply.rc == 0 + when: "'field is immutable' in generation_apply.stderr" diff --git a/roles/openshift_logging/tasks/oc_secret.yaml b/roles/openshift_logging/tasks/oc_secret.yaml new file mode 100644 index 000000000..de37e4f6d --- /dev/null +++ b/roles/openshift_logging/tasks/oc_secret.yaml @@ -0,0 +1,7 @@ +--- +- command: > + {{ openshift.common.client_binary }} + --config={{ kubeconfig }} + secret {{subcommand}} {{service_account}} {{secret_name}} + {{add_args}} + -n {{openshift_logging_namespace}} diff --git a/roles/openshift_logging/tasks/start_cluster.yaml b/roles/openshift_logging/tasks/start_cluster.yaml index 07489ae79..69d2b2b6b 100644 --- a/roles/openshift_logging/tasks/start_cluster.yaml +++ b/roles/openshift_logging/tasks/start_cluster.yaml @@ -86,7 +86,7 @@ with_items: "{{es_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool - command: > {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=kibana-ops -o name -n {{openshift_logging_namespace}} @@ -104,7 +104,7 @@ with_items: "{{kibana_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool - command: > {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=curator-ops -o name -n {{openshift_logging_namespace}} @@ -122,4 +122,4 @@ with_items: "{{curator_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool diff --git a/roles/openshift_logging/tasks/stop_cluster.yaml b/roles/openshift_logging/tasks/stop_cluster.yaml index 8e0df8344..7826efabe 100644 --- a/roles/openshift_logging/tasks/stop_cluster.yaml +++ b/roles/openshift_logging/tasks/stop_cluster.yaml @@ -81,7 +81,7 @@ with_items: "{{es_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool - command: > {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=kibana-ops -o name -n {{openshift_logging_namespace}} @@ -98,7 +98,7 @@ with_items: "{{kibana_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool - command: > {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig get dc -l component=curator-ops -o name -n {{openshift_logging_namespace}} @@ -115,4 +115,4 @@ with_items: "{{curator_dc.stdout_lines}}" loop_control: loop_var: object - when: openshift_logging_use_ops + when: openshift_logging_use_ops | bool diff --git a/roles/openshift_master/templates/master.yaml.v1.j2 b/roles/openshift_master/templates/master.yaml.v1.j2 index cf7ceacff..9ae54dac1 100644 --- a/roles/openshift_master/templates/master.yaml.v1.j2 +++ b/roles/openshift_master/templates/master.yaml.v1.j2 @@ -102,7 +102,11 @@ imagePolicyConfig:{{ openshift.master.image_policy_config | to_padded_yaml(level kind: MasterConfig kubeletClientInfo: {# TODO: allow user specified kubelet port #} +{% if openshift.common.version_gte_3_2_or_1_2 | bool %} + ca: ca-bundle.crt +{% else %} ca: ca.crt +{% endif %} certFile: master.kubelet-client.crt keyFile: master.kubelet-client.key port: 10250 @@ -221,7 +225,11 @@ servingInfo: bindAddress: {{ openshift.master.bind_addr }}:{{ openshift.master.api_port }} bindNetwork: tcp4 certFile: master.server.crt +{% if openshift.common.version_gte_3_2_or_1_2 | bool %} + clientCA: ca-bundle.crt +{% else %} clientCA: ca.crt +{% endif %} keyFile: master.server.key maxRequestsInFlight: {{ openshift.master.max_requests_inflight }} requestTimeoutSeconds: 3600 diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml index 4620dd877..7a5ed51ec 100644 --- a/roles/openshift_master_certificates/tasks/main.yml +++ b/roles/openshift_master_certificates/tasks/main.yml @@ -38,12 +38,22 @@ when: master_certs_missing | bool and inventory_hostname != openshift_ca_host delegate_to: "{{ openshift_ca_host }}" +- find: + paths: "{{ openshift_master_config_dir }}/legacy-ca/" + patterns: ".*-ca.crt" + use_regex: true + register: g_master_legacy_ca_result + delegate_to: "{{ openshift_ca_host }}" + - name: Create the master server certificate command: > {{ hostvars[openshift_ca_host].openshift.common.client_binary }} adm ca create-server-cert {% for named_ca_certificate in openshift.master.named_certificates | default([]) | oo_collect('cafile') %} --certificate-authority {{ named_ca_certificate }} {% endfor %} + {% for legacy_ca_certificate in g_master_legacy_ca_result.files | default([]) | oo_collect('path') %} + --certificate-authority {{ legacy_ca_certificate }} + {% endfor %} --hostnames={{ hostvars[item].openshift.common.all_hostnames | join(',') }} --cert={{ openshift_generated_configs_dir }}/master-{{ hostvars[item].openshift.common.hostname }}/master.server.crt --key={{ openshift_generated_configs_dir }}/master-{{ hostvars[item].openshift.common.hostname }}/master.server.key diff --git a/roles/openshift_metrics/tasks/install_support.yaml b/roles/openshift_metrics/tasks/install_support.yaml index cc5acc6e5..5cefb273d 100644 --- a/roles/openshift_metrics/tasks/install_support.yaml +++ b/roles/openshift_metrics/tasks/install_support.yaml @@ -9,7 +9,7 @@ when: htpasswd_check.rc == 1 - name: Check control node to see if keytool is installed - local_action: command which htpasswd + local_action: command which keytool register: keytool_check failed_when: no changed_when: no diff --git a/roles/openshift_node_certificates/handlers/main.yml b/roles/openshift_node_certificates/handlers/main.yml index a74668b13..1aa826c09 100644 --- a/roles/openshift_node_certificates/handlers/main.yml +++ b/roles/openshift_node_certificates/handlers/main.yml @@ -8,3 +8,4 @@ systemd: name: docker state: restarted + when: not openshift_certificates_redeploy | default(false) | bool diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml index a263f4f3a..4cb89aba2 100644 --- a/roles/openshift_node_certificates/tasks/main.yml +++ b/roles/openshift_node_certificates/tasks/main.yml @@ -42,20 +42,30 @@ when: node_certs_missing | bool delegate_to: "{{ openshift_ca_host }}" +- find: + paths: "{{ openshift.common.config_base }}/master/legacy-ca/" + patterns: ".*-ca.crt" + use_regex: true + register: g_master_legacy_ca_result + delegate_to: "{{ openshift_ca_host }}" + - name: Generate the node client config command: > {{ hostvars[openshift_ca_host].openshift.common.client_binary }} adm create-api-client-config - {% for named_ca_certificate in hostvars[openshift_ca_host].openshift.master.named_certificates | default([]) | oo_collect('cafile') %} - --certificate-authority {{ named_ca_certificate }} - {% endfor %} - --certificate-authority={{ openshift_ca_cert }} - --client-dir={{ openshift_generated_configs_dir }}/node-{{ hostvars[item].openshift.common.hostname }} - --groups=system:nodes - --master={{ hostvars[openshift_ca_host].openshift.master.api_url }} - --signer-cert={{ openshift_ca_cert }} - --signer-key={{ openshift_ca_key }} - --signer-serial={{ openshift_ca_serial }} - --user=system:node:{{ hostvars[item].openshift.common.hostname }} + {% for named_ca_certificate in hostvars[openshift_ca_host].openshift.master.named_certificates | default([]) | oo_collect('cafile') %} + --certificate-authority {{ named_ca_certificate }} + {% endfor %} + {% for legacy_ca_certificate in g_master_legacy_ca_result.files | default([]) | oo_collect('path') %} + --certificate-authority {{ legacy_ca_certificate }} + {% endfor %} + --certificate-authority={{ openshift_ca_cert }} + --client-dir={{ openshift_generated_configs_dir }}/node-{{ hostvars[item].openshift.common.hostname }} + --groups=system:nodes + --master={{ hostvars[openshift_ca_host].openshift.master.api_url }} + --signer-cert={{ openshift_ca_cert }} + --signer-key={{ openshift_ca_key }} + --signer-serial={{ openshift_ca_serial }} + --user=system:node:{{ hostvars[item].openshift.common.hostname }} args: creates: "{{ openshift_generated_configs_dir }}/node-{{ hostvars[item].openshift.common.hostname }}" with_items: "{{ hostvars diff --git a/roles/openshift_node_upgrade/README.md b/roles/openshift_node_upgrade/README.md new file mode 100644 index 000000000..e21bee412 --- /dev/null +++ b/roles/openshift_node_upgrade/README.md @@ -0,0 +1,108 @@ +OpenShift/Atomic Enterprise Node upgrade +========= + +Role responsible for a single node upgrade. +It is expected a node is functioning and a part of an OpenShift cluster. + +Requirements +------------ + +TODO + +Role Variables +-------------- +From this role: + +| Name | Default value | | +|--------------------------------|-----------------------|--------------------------------------------------------| +| deployment_type | | Inventory var | +| docker_upgrade_nuke_images | | Optional inventory var | +| docker_version | | Optional inventory var | +| l_docker_upgrade | | | +| node_config_hook | | | +| openshift.docker.gte_1_10 | | | +| openshift_image_tag | | Set by openshift_version role | +| openshift_pkg_version | | Set by openshift_version role | +| openshift_release | | Set by openshift_version role | +| skip_docker_restart | | | +| openshift_cloudprovider_kind | | | + +From openshift.common: + +| Name | Default Value | | +|------------------------------------|---------------------|---------------------| +| openshift.common.config_base |---------------------|---------------------| +| openshift.common.data_dir |---------------------|---------------------| +| openshift.common.hostname |---------------------|---------------------| +| openshift.common.http_proxy |---------------------|---------------------| +| openshift.common.is_atomic |---------------------|---------------------| +| openshift.common.is_containerized |---------------------|---------------------| +| openshift.common.portal_net |---------------------|---------------------| +| openshift.common.service_type |---------------------|---------------------| +| openshift.common.use_openshift_sdn |---------------------|---------------------| + +From openshift.master: + +| Name | Default Value | | +|------------------------------------|---------------------|---------------------| +| openshift.master.api_port |---------------------|---------------------| + +From openshift.node: + +| Name | Default Value | | +|------------------------------------|---------------------|---------------------| +| openshift.node.debug_level |---------------------|---------------------| +| openshift.node.node_image |---------------------|---------------------| +| openshift.node.ovs_image |---------------------|---------------------| + + +Dependencies +------------ +openshift_common + +TODO + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + +``` +--- +- name: Upgrade nodes + hosts: oo_nodes_to_upgrade + serial: 1 + any_errors_fatal: true + + pre_tasks: + - name: Mark unschedulable + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + + - name: Drain Node for Kubelet upgrade + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data + delegate_to: "{{ groups.oo_first_master.0 }}" + + roles: + - openshift_facts + - docker + - openshift_node_upgrade + + post_tasks: + - name: Set node schedulability + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} adm manage-node {{ openshift.node.nodename | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" +``` + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +TODO diff --git a/roles/openshift_node_upgrade/files/nuke_images.sh b/roles/openshift_node_upgrade/files/nuke_images.sh new file mode 100644 index 000000000..8635eab0d --- /dev/null +++ b/roles/openshift_node_upgrade/files/nuke_images.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Stop any running containers +running_container_ids=`docker ps -q` +if test -n "$running_container_ids" +then + docker stop $running_container_ids +fi + +# Delete all containers +container_ids=`docker ps -a -q` +if test -n "$container_ids" +then + docker rm -f -v $container_ids +fi + +# Delete all images (forcefully) +image_ids=`docker images -aq` +if test -n "$image_ids" +then + # Some layers are deleted recursively and are no longer present + # when docker goes to remove them: + docker rmi -f `docker images -aq` || true +fi + diff --git a/roles/openshift_node_upgrade/handlers/main.yml b/roles/openshift_node_upgrade/handlers/main.yml new file mode 100644 index 000000000..cb51416d4 --- /dev/null +++ b/roles/openshift_node_upgrade/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: restart openvswitch + systemd: name=openvswitch state=restarted + when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift.common.use_openshift_sdn | bool + notify: + - restart openvswitch pause + +- name: restart openvswitch pause + pause: seconds=15 + when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool + +- name: restart node + systemd: name={{ openshift.common.service_type }}-node state=restarted + when: (not skip_node_svc_handlers | default(False) | bool) and not (node_service_status_changed | default(false) | bool) diff --git a/roles/openshift_node_upgrade/meta/main.yml b/roles/openshift_node_upgrade/meta/main.yml new file mode 100644 index 000000000..cd2f362aa --- /dev/null +++ b/roles/openshift_node_upgrade/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: your name + description: OpenShift Node upgrade + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 2.1 + platforms: + - name: EL + versions: + - 7 +dependencies: +- role: openshift_common diff --git a/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml new file mode 100644 index 000000000..07b0ac715 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml @@ -0,0 +1,14 @@ +--- +# This is a hack to allow us to use systemd_units.yml, but skip the handlers which +# restart services. We will unconditionally restart all containerized services +# because we have to unconditionally restart Docker: +- set_fact: + skip_node_svc_handlers: True + +- name: Update systemd units + include: systemd_units.yml + +# This is a no-op because of skip_node_svc_handlers, but lets us trigger it before end of +# play when the node has already been marked schedulable again. (this would look strange +# in logs otherwise) +- meta: flush_handlers diff --git a/roles/openshift_node_upgrade/tasks/docker/restart.yml b/roles/openshift_node_upgrade/tasks/docker/restart.yml new file mode 100644 index 000000000..176fc3c0b --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/docker/restart.yml @@ -0,0 +1,33 @@ +--- +# input variables: +# - openshift.common.service_type +# - openshift.common.is_containerized +# - openshift.common.hostname +# - openshift.master.api_port + +- name: Restart docker + service: name=docker state=restarted + +- name: Update docker facts + openshift_facts: + role: docker + +- name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Wait for master API to come back online + wait_for: + host: "{{ openshift.common.hostname }}" + state: started + delay: 10 + port: "{{ openshift.master.api_port }}" + when: inventory_hostname in groups.oo_masters_to_config diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml new file mode 100644 index 000000000..e91891ca9 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml @@ -0,0 +1,49 @@ +--- +# input variables: +# - openshift.common.service_type +# - openshift.common.is_containerized +# - docker_upgrade_nuke_images +# - docker_version +# - skip_docker_restart + +# We need docker service up to remove all the images, but these services will keep +# trying to re-start and thus re-pull the images we're trying to delete. +- name: Stop containerized services + service: name={{ item }} state=stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + - etcd_container + - openvswitch + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + +- debug: var=docker_image_count.stdout + +# TODO(jchaloup): put all docker_upgrade_nuke_images into a block with only one condition +- name: Remove all containers and images + script: nuke_images.sh + register: nuke_images_result + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- name: Check Docker image count + shell: "docker images -aq | wc -l" + register: docker_image_count + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- debug: var=docker_image_count.stdout + when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool + +- service: name=docker state=stopped + +- name: Upgrade Docker + package: name=docker{{ '-' + docker_version }} state=present + +- include: restart.yml + when: not skip_docker_restart | default(False) | bool diff --git a/roles/openshift_node_upgrade/tasks/main.yml b/roles/openshift_node_upgrade/tasks/main.yml new file mode 100644 index 000000000..b1d5f0e0f --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/main.yml @@ -0,0 +1,77 @@ +--- +# input variables: +# - l_docker_upgrade +# - openshift.common.is_atomic +# - node_config_hook +# - openshift_pkg_version +# - openshift.common.is_containerized +# - deployment_type +# - openshift_release + +# tasks file for openshift_node_upgrade +- include: docker/upgrade.yml + vars: + # We will restart Docker ourselves after everything is ready: + skip_docker_restart: True + when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool + +- include: "{{ node_config_hook }}" + when: node_config_hook is defined + +- include: rpm_upgrade.yml + vars: + component: "node" + openshift_version: "{{ openshift_pkg_version | default('') }}" + when: not openshift.common.is_containerized | bool + +- name: Remove obsolete docker-sdn-ovs.conf + file: path=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf state=absent + when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>=')) + +- include: containerized_node_upgrade.yml + when: openshift.common.is_containerized | bool + +- name: Ensure containerized services stopped before Docker restart + service: name={{ item }} state=stopped + with_items: + - etcd_container + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: openshift.common.is_containerized | bool + +- name: Upgrade openvswitch + package: + name: openvswitch + state: latest + register: ovs_pkg + when: not openshift.common.is_containerized | bool + +- name: Restart openvswitch + systemd: + name: openvswitch + state: restarted + when: + - not openshift.common.is_containerized | bool + - ovs_pkg | changed + +# Mandatory Docker restart, ensure all containerized services are running: +- include: docker/restart.yml + +- name: Restart rpm node service + service: name="{{ openshift.common.service_type }}-node" state=restarted + when: not openshift.common.is_containerized | bool + +- name: Wait for node to be ready + command: > + {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers + register: node_output + delegate_to: "{{ groups.oo_first_master.0 }}" + until: "{{ node_output.stdout.split()[1].startswith('Ready')}}" + # Give the node two minutes to come back online. Note that we pre-pull images now + # so containerized services should restart quickly as well. + retries: 24 + delay: 5 diff --git a/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml new file mode 100644 index 000000000..480e87d58 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml @@ -0,0 +1,14 @@ +--- +# input variables: +# - openshift.common.service_type +# - component +# - openshift_pkg_version +# - openshift.common.is_atomic + +# We verified latest rpm available is suitable, so just yum update. +- name: Upgrade packages + package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present" + +- name: Ensure python-yaml present for config upgrade + package: name=PyYAML state=present + when: not openshift.common.is_atomic | bool diff --git a/roles/openshift_node_upgrade/tasks/systemd_units.yml b/roles/openshift_node_upgrade/tasks/systemd_units.yml new file mode 100644 index 000000000..862cd19c4 --- /dev/null +++ b/roles/openshift_node_upgrade/tasks/systemd_units.yml @@ -0,0 +1,119 @@ +--- +# input variables +# - openshift.node.node_image +# - openshift_image_tag +# - openshift.common.is_containerized +# - openshift.node.ovs_image +# - openshift.common.use_openshift_sdn +# - openshift.common.service_type +# - openshift.node.debug_level +# - openshift.common.config_base +# - openshift.common.http_proxy +# - openshift.common.portal_net +# - openshift.common +# - openshift.common.http_proxy +# notify: +# - restart openvswitch +# - restart node + +# This file is included both in the openshift_master role and in the upgrade +# playbooks. + +- name: Pre-pull node image + command: > + docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool + +- name: Pre-pull openvswitch image + command: > + docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }} + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout" + when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool + +- name: Install Node dependencies docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service" + src: openshift.docker.node.dep.service + register: install_node_dep_result + when: openshift.common.is_containerized | bool + +- name: Install Node docker service file + template: + dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service" + src: openshift.docker.node.service + register: install_node_result + when: openshift.common.is_containerized | bool + +- name: Create the openvswitch service env file + template: + src: openvswitch.sysconfig.j2 + dest: /etc/sysconfig/openvswitch + when: openshift.common.is_containerized | bool + register: install_ovs_sysconfig + notify: + - restart openvswitch + +# May be a temporary workaround. +# https://bugzilla.redhat.com/show_bug.cgi?id=1331590 +- name: Create OpenvSwitch service.d directory + file: path=/etc/systemd/system/openvswitch.service.d/ state=directory + when: openshift.common.use_openshift_sdn | default(true) | bool + +- name: Install OpenvSwitch service OOM fix + template: + dest: "/etc/systemd/system/openvswitch.service.d/01-avoid-oom.conf" + src: openvswitch-avoid-oom.conf + when: openshift.common.use_openshift_sdn | default(true) | bool + register: install_oom_fix_result + notify: + - restart openvswitch + +- name: Install OpenvSwitch docker service file + template: + dest: "/etc/systemd/system/openvswitch.service" + src: openvswitch.docker.service + when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | default(true) | bool + notify: + - restart openvswitch + +- name: Configure Node settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^OPTIONS=' + line: "OPTIONS=--loglevel={{ openshift.node.debug_level | default(2) }}" + - regex: '^CONFIG_FILE=' + line: "CONFIG_FILE={{ openshift.common.config_base }}/node/node-config.yaml" + - regex: '^IMAGE_VERSION=' + line: "IMAGE_VERSION={{ openshift_image_tag }}" + notify: + - restart node + +- name: Configure Proxy Settings + lineinfile: + dest: /etc/sysconfig/{{ openshift.common.service_type }}-node + regexp: "{{ item.regex }}" + line: "{{ item.line }}" + create: true + with_items: + - regex: '^HTTP_PROXY=' + line: "HTTP_PROXY={{ openshift.common.http_proxy | default('') }}" + - regex: '^HTTPS_PROXY=' + line: "HTTPS_PROXY={{ openshift.common.https_proxy | default('') }}" + - regex: '^NO_PROXY=' + line: "NO_PROXY={{ openshift.common.no_proxy | default([]) }},{{ openshift.common.portal_net }},{{ hostvars[groups.oo_first_master.0].openshift.master.sdn_cluster_network_cidr }}" + when: ('http_proxy' in openshift.common and openshift.common.http_proxy != '') + notify: + - restart node + +- name: Reload systemd units + command: systemctl daemon-reload + when: (openshift.common.is_containerized | bool and (install_node_result | changed or install_ovs_sysconfig | changed or install_node_dep_result | changed)) or install_oom_fix_result | changed + notify: + - restart node diff --git a/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service b/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service new file mode 100644 index 000000000..0fb34cffd --- /dev/null +++ b/roles/openshift_node_upgrade/templates/openshift.docker.node.dep.service @@ -0,0 +1,11 @@ +[Unit] +Requires=docker.service +After=docker.service +PartOf={{ openshift.common.service_type }}-node.service +Before={{ openshift.common.service_type }}-node.service + + +[Service] +ExecStart=/bin/bash -c "if [[ -f /usr/bin/docker-current ]]; then echo \"DOCKER_ADDTL_BIND_MOUNTS=--volume=/usr/bin/docker-current:/usr/bin/docker-current:ro --volume=/etc/sysconfig/docker:/etc/sysconfig/docker:ro\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; else echo \"#DOCKER_ADDTL_BIND_MOUNTS=\" > /etc/sysconfig/{{ openshift.common.service_type }}-node-dep; fi" +ExecStop= +SyslogIdentifier={{ openshift.common.service_type }}-node-dep diff --git a/roles/openshift_node_upgrade/templates/openshift.docker.node.service b/roles/openshift_node_upgrade/templates/openshift.docker.node.service new file mode 100644 index 000000000..e33d5d497 --- /dev/null +++ b/roles/openshift_node_upgrade/templates/openshift.docker.node.service @@ -0,0 +1,26 @@ +[Unit] +After={{ openshift.common.service_type }}-master.service +After=docker.service +After=openvswitch.service +PartOf=docker.service +Requires=docker.service +{% if openshift.common.use_openshift_sdn %} +Requires=openvswitch.service +{% endif %} +Wants={{ openshift.common.service_type }}-master.service +Requires={{ openshift.common.service_type }}-node-dep.service +After={{ openshift.common.service_type }}-node-dep.service + +[Service] +EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node +EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node-dep +ExecStartPre=-/usr/bin/docker rm -f {{ openshift.common.service_type }}-node +ExecStart=/usr/bin/docker run --name {{ openshift.common.service_type }}-node --rm --privileged --net=host --pid=host --env-file=/etc/sysconfig/{{ openshift.common.service_type }}-node -v /:/rootfs:ro -e CONFIG_FILE=${CONFIG_FILE} -e OPTIONS=${OPTIONS} -e HOST=/rootfs -e HOST_ETC=/host-etc -v {{ openshift.common.data_dir }}:{{ openshift.common.data_dir }}{{ ':rslave' if openshift.docker.gte_1_10 | default(False) | bool else '' }} -v {{ openshift.common.config_base }}/node:{{ openshift.common.config_base }}/node {% if openshift_cloudprovider_kind | default('') != '' -%} -v {{ openshift.common.config_base }}/cloudprovider:{{ openshift.common.config_base}}/cloudprovider {% endif -%} -v /etc/localtime:/etc/localtime:ro -v /etc/machine-id:/etc/machine-id:ro -v /run:/run -v /sys:/sys:rw -v /sys/fs/cgroup:/sys/fs/cgroup:rw -v /usr/bin/docker:/usr/bin/docker:ro -v /var/lib/docker:/var/lib/docker -v /lib/modules:/lib/modules -v /etc/origin/openvswitch:/etc/openvswitch -v /etc/origin/sdn:/etc/openshift-sdn -v /etc/systemd/system:/host-etc/systemd/system -v /var/log:/var/log -v /dev:/dev $DOCKER_ADDTL_BIND_MOUNTS {{ openshift.node.node_image }}:${IMAGE_VERSION} +ExecStartPost=/usr/bin/sleep 10 +ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node +SyslogIdentifier={{ openshift.common.service_type }}-node +Restart=always +RestartSec=5s + +[Install] +WantedBy=docker.service diff --git a/roles/openshift_node_upgrade/templates/openvswitch-avoid-oom.conf b/roles/openshift_node_upgrade/templates/openvswitch-avoid-oom.conf new file mode 100644 index 000000000..3229bc56b --- /dev/null +++ b/roles/openshift_node_upgrade/templates/openvswitch-avoid-oom.conf @@ -0,0 +1,3 @@ +# Avoid the OOM killer for openvswitch and it's children: +[Service] +OOMScoreAdjust=-1000 diff --git a/roles/openshift_node_upgrade/templates/openvswitch.docker.service b/roles/openshift_node_upgrade/templates/openvswitch.docker.service new file mode 100644 index 000000000..1e1f8967d --- /dev/null +++ b/roles/openshift_node_upgrade/templates/openvswitch.docker.service @@ -0,0 +1,17 @@ +[Unit] +After=docker.service +Requires=docker.service +PartOf=docker.service + +[Service] +EnvironmentFile=/etc/sysconfig/openvswitch +ExecStartPre=-/usr/bin/docker rm -f openvswitch +ExecStart=/usr/bin/docker run --name openvswitch --rm --privileged --net=host --pid=host -v /lib/modules:/lib/modules -v /run:/run -v /sys:/sys:ro -v /etc/origin/openvswitch:/etc/openvswitch {{ openshift.node.ovs_image }}:${IMAGE_VERSION} +ExecStartPost=/usr/bin/sleep 5 +ExecStop=/usr/bin/docker stop openvswitch +SyslogIdentifier=openvswitch +Restart=always +RestartSec=5s + +[Install] +WantedBy=docker.service diff --git a/roles/openshift_node_upgrade/templates/openvswitch.sysconfig.j2 b/roles/openshift_node_upgrade/templates/openvswitch.sysconfig.j2 new file mode 100644 index 000000000..da7c3742a --- /dev/null +++ b/roles/openshift_node_upgrade/templates/openvswitch.sysconfig.j2 @@ -0,0 +1 @@ +IMAGE_VERSION={{ openshift_image_tag }} |