From 79939ca092b9ee0fe31e5964102c60fca73da568 Mon Sep 17 00:00:00 2001 From: Russell Teague Date: Thu, 16 Nov 2017 11:31:29 -0500 Subject: Playbook Consolidation - openshift-etcd --- playbooks/aws/openshift-cluster/install.yml | 2 +- .../openshift-cluster/redeploy-certificates.yml | 4 +- .../redeploy-etcd-certificates.yml | 4 +- playbooks/byo/openshift-etcd/certificates.yml | 6 - playbooks/byo/openshift-etcd/config.yml | 4 - playbooks/byo/openshift-etcd/embedded2external.yml | 4 - playbooks/byo/openshift-etcd/filter_plugins | 1 - playbooks/byo/openshift-etcd/lookup_plugins | 1 - playbooks/byo/openshift-etcd/migrate.yml | 4 - playbooks/byo/openshift-etcd/restart.yml | 4 - playbooks/byo/openshift-etcd/roles | 1 - playbooks/byo/openshift-etcd/scaleup.yml | 4 - playbooks/common/openshift-cluster/config.yml | 2 +- .../redeploy-certificates/etcd-ca.yml | 4 +- playbooks/common/openshift-etcd/ca.yml | 15 -- playbooks/common/openshift-etcd/certificates.yml | 4 - playbooks/common/openshift-etcd/config.yml | 40 ----- .../common/openshift-etcd/embedded2external.yml | 172 --------------------- playbooks/common/openshift-etcd/filter_plugins | 1 - playbooks/common/openshift-etcd/lookup_plugins | 1 - .../openshift-etcd/master_etcd_certificates.yml | 14 -- playbooks/common/openshift-etcd/migrate.yml | 169 -------------------- playbooks/common/openshift-etcd/restart.yml | 27 ---- playbooks/common/openshift-etcd/roles | 1 - playbooks/common/openshift-etcd/scaleup.yml | 83 ---------- .../common/openshift-etcd/server_certificates.yml | 15 -- playbooks/common/openshift-master/scaleup.yml | 2 +- playbooks/openshift-etcd/certificates.yml | 6 + playbooks/openshift-etcd/config.yml | 4 + playbooks/openshift-etcd/embedded2external.yml | 4 + playbooks/openshift-etcd/migrate.yml | 4 + playbooks/openshift-etcd/private/ca.yml | 15 ++ playbooks/openshift-etcd/private/certificates.yml | 4 + playbooks/openshift-etcd/private/config.yml | 40 +++++ .../openshift-etcd/private/embedded2external.yml | 172 +++++++++++++++++++++ playbooks/openshift-etcd/private/filter_plugins | 1 + playbooks/openshift-etcd/private/lookup_plugins | 1 + .../private/master_etcd_certificates.yml | 14 ++ playbooks/openshift-etcd/private/migrate.yml | 169 ++++++++++++++++++++ playbooks/openshift-etcd/private/restart.yml | 27 ++++ playbooks/openshift-etcd/private/roles | 1 + playbooks/openshift-etcd/private/scaleup.yml | 83 ++++++++++ .../openshift-etcd/private/server_certificates.yml | 15 ++ playbooks/openshift-etcd/restart.yml | 4 + playbooks/openshift-etcd/scaleup.yml | 4 + 45 files changed, 577 insertions(+), 580 deletions(-) delete mode 100644 playbooks/byo/openshift-etcd/certificates.yml delete mode 100644 playbooks/byo/openshift-etcd/config.yml delete mode 100644 playbooks/byo/openshift-etcd/embedded2external.yml delete mode 120000 playbooks/byo/openshift-etcd/filter_plugins delete mode 120000 playbooks/byo/openshift-etcd/lookup_plugins delete mode 100644 playbooks/byo/openshift-etcd/migrate.yml delete mode 100644 playbooks/byo/openshift-etcd/restart.yml delete mode 120000 playbooks/byo/openshift-etcd/roles delete mode 100644 playbooks/byo/openshift-etcd/scaleup.yml delete mode 100644 playbooks/common/openshift-etcd/ca.yml delete mode 100644 playbooks/common/openshift-etcd/certificates.yml delete mode 100644 playbooks/common/openshift-etcd/config.yml delete mode 100644 playbooks/common/openshift-etcd/embedded2external.yml delete mode 120000 playbooks/common/openshift-etcd/filter_plugins delete mode 120000 playbooks/common/openshift-etcd/lookup_plugins delete mode 100644 playbooks/common/openshift-etcd/master_etcd_certificates.yml delete mode 100644 playbooks/common/openshift-etcd/migrate.yml delete mode 100644 playbooks/common/openshift-etcd/restart.yml delete mode 120000 playbooks/common/openshift-etcd/roles delete mode 100644 playbooks/common/openshift-etcd/scaleup.yml delete mode 100644 playbooks/common/openshift-etcd/server_certificates.yml create mode 100644 playbooks/openshift-etcd/certificates.yml create mode 100644 playbooks/openshift-etcd/config.yml create mode 100644 playbooks/openshift-etcd/embedded2external.yml create mode 100644 playbooks/openshift-etcd/migrate.yml create mode 100644 playbooks/openshift-etcd/private/ca.yml create mode 100644 playbooks/openshift-etcd/private/certificates.yml create mode 100644 playbooks/openshift-etcd/private/config.yml create mode 100644 playbooks/openshift-etcd/private/embedded2external.yml create mode 120000 playbooks/openshift-etcd/private/filter_plugins create mode 120000 playbooks/openshift-etcd/private/lookup_plugins create mode 100644 playbooks/openshift-etcd/private/master_etcd_certificates.yml create mode 100644 playbooks/openshift-etcd/private/migrate.yml create mode 100644 playbooks/openshift-etcd/private/restart.yml create mode 120000 playbooks/openshift-etcd/private/roles create mode 100644 playbooks/openshift-etcd/private/scaleup.yml create mode 100644 playbooks/openshift-etcd/private/server_certificates.yml create mode 100644 playbooks/openshift-etcd/restart.yml create mode 100644 playbooks/openshift-etcd/scaleup.yml (limited to 'playbooks') diff --git a/playbooks/aws/openshift-cluster/install.yml b/playbooks/aws/openshift-cluster/install.yml index 8756fb52a..6c0eefaf9 100644 --- a/playbooks/aws/openshift-cluster/install.yml +++ b/playbooks/aws/openshift-cluster/install.yml @@ -22,7 +22,7 @@ include: ../../common/openshift-checks/install.yml - name: etcd install - include: ../../common/openshift-etcd/config.yml + include: ../../openshift-etcd/private/config.yml - name: include nfs include: ../../common/openshift-nfs/config.yml diff --git a/playbooks/byo/openshift-cluster/redeploy-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-certificates.yml index 6450a4d76..9e52a054d 100644 --- a/playbooks/byo/openshift-cluster/redeploy-certificates.yml +++ b/playbooks/byo/openshift-cluster/redeploy-certificates.yml @@ -7,7 +7,7 @@ - include: ../../common/openshift-cluster/redeploy-certificates/etcd-backup.yml -- include: ../../common/openshift-etcd/certificates.yml +- include: ../../openshift-etcd/private/certificates.yml vars: etcd_certificates_redeploy: true @@ -23,7 +23,7 @@ vars: openshift_certificates_redeploy: true -- include: ../../common/openshift-etcd/restart.yml +- include: ../../openshift-etcd/private/restart.yml vars: g_etcd_certificates_expired: "{{ ('expired' in (hostvars | oo_select_keys(groups['etcd']) | oo_collect('check_results.check_results.etcd') | oo_collect('health'))) | bool }}" diff --git a/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml b/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml index f3892f56f..f9d12251f 100644 --- a/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml +++ b/playbooks/byo/openshift-cluster/redeploy-etcd-certificates.yml @@ -7,11 +7,11 @@ - include: ../../common/openshift-cluster/redeploy-certificates/etcd-backup.yml -- include: ../../common/openshift-etcd/certificates.yml +- include: ../../openshift-etcd/private/certificates.yml vars: etcd_certificates_redeploy: true -- include: ../../common/openshift-etcd/restart.yml +- include: ../../openshift-etcd/private/restart.yml vars: g_etcd_certificates_expired: "{{ ('expired' in (hostvars | oo_select_keys(groups['etcd']) | oo_collect('check_results.check_results.etcd') | oo_collect('health'))) | bool }}" diff --git a/playbooks/byo/openshift-etcd/certificates.yml b/playbooks/byo/openshift-etcd/certificates.yml deleted file mode 100644 index 0e9d42cd6..000000000 --- a/playbooks/byo/openshift-etcd/certificates.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/ca.yml - -- include: ../../common/openshift-etcd/certificates.yml diff --git a/playbooks/byo/openshift-etcd/config.yml b/playbooks/byo/openshift-etcd/config.yml deleted file mode 100644 index c6e0a9d90..000000000 --- a/playbooks/byo/openshift-etcd/config.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/config.yml diff --git a/playbooks/byo/openshift-etcd/embedded2external.yml b/playbooks/byo/openshift-etcd/embedded2external.yml deleted file mode 100644 index 492f677b0..000000000 --- a/playbooks/byo/openshift-etcd/embedded2external.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/embedded2external.yml diff --git a/playbooks/byo/openshift-etcd/filter_plugins b/playbooks/byo/openshift-etcd/filter_plugins deleted file mode 120000 index 99a95e4ca..000000000 --- a/playbooks/byo/openshift-etcd/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../filter_plugins \ No newline at end of file diff --git a/playbooks/byo/openshift-etcd/lookup_plugins b/playbooks/byo/openshift-etcd/lookup_plugins deleted file mode 120000 index ac79701db..000000000 --- a/playbooks/byo/openshift-etcd/lookup_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../lookup_plugins \ No newline at end of file diff --git a/playbooks/byo/openshift-etcd/migrate.yml b/playbooks/byo/openshift-etcd/migrate.yml deleted file mode 100644 index 3020e7db4..000000000 --- a/playbooks/byo/openshift-etcd/migrate.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/migrate.yml diff --git a/playbooks/byo/openshift-etcd/restart.yml b/playbooks/byo/openshift-etcd/restart.yml deleted file mode 100644 index 0889d91ba..000000000 --- a/playbooks/byo/openshift-etcd/restart.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/restart.yml diff --git a/playbooks/byo/openshift-etcd/roles b/playbooks/byo/openshift-etcd/roles deleted file mode 120000 index 20c4c58cf..000000000 --- a/playbooks/byo/openshift-etcd/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles \ No newline at end of file diff --git a/playbooks/byo/openshift-etcd/scaleup.yml b/playbooks/byo/openshift-etcd/scaleup.yml deleted file mode 100644 index e7d62e264..000000000 --- a/playbooks/byo/openshift-etcd/scaleup.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-etcd/scaleup.yml diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml index 3b4d6f9a6..6b78493b9 100644 --- a/playbooks/common/openshift-cluster/config.yml +++ b/playbooks/common/openshift-cluster/config.yml @@ -1,7 +1,7 @@ --- - include: ../openshift-checks/install.yml -- include: ../openshift-etcd/config.yml +- include: ../../openshift-etcd/private/config.yml - include: ../openshift-nfs/config.yml when: groups.oo_nfs_to_config | default([]) | count > 0 diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/etcd-ca.yml b/playbooks/common/openshift-cluster/redeploy-certificates/etcd-ca.yml index 044875d1c..41e19f5d6 100644 --- a/playbooks/common/openshift-cluster/redeploy-certificates/etcd-ca.yml +++ b/playbooks/common/openshift-cluster/redeploy-certificates/etcd-ca.yml @@ -21,7 +21,7 @@ name: etcd tasks_from: remove_ca_certificates -- include: ../../openshift-etcd/ca.yml +- include: ../../../openshift-etcd/private/ca.yml - name: Create temp directory for syncing certs hosts: localhost @@ -45,7 +45,7 @@ etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" -- include: ../../openshift-etcd/restart.yml +- include: ../../../openshift-etcd/private/restart.yml # Do not restart etcd when etcd certificates were previously expired. when: ('expired' not in (hostvars | oo_select_keys(groups['etcd']) diff --git a/playbooks/common/openshift-etcd/ca.yml b/playbooks/common/openshift-etcd/ca.yml deleted file mode 100644 index ac5543be9..000000000 --- a/playbooks/common/openshift-etcd/ca.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Generate new etcd CA - hosts: oo_first_etcd - roles: - - role: openshift_etcd_facts - tasks: - - include_role: - name: etcd - tasks_from: ca - vars: - etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" - when: - - etcd_ca_setup | default(True) | bool diff --git a/playbooks/common/openshift-etcd/certificates.yml b/playbooks/common/openshift-etcd/certificates.yml deleted file mode 100644 index eb6b94f33..000000000 --- a/playbooks/common/openshift-etcd/certificates.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: server_certificates.yml - -- include: master_etcd_certificates.yml diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml deleted file mode 100644 index 3fe483785..000000000 --- a/playbooks/common/openshift-etcd/config.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: etcd Install Checkpoint Start - hosts: all - gather_facts: false - tasks: - - name: Set etcd install 'In Progress' - run_once: true - set_stats: - data: - installer_phase_etcd: - status: "In Progress" - start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - -- include: ca.yml - -- include: certificates.yml - -- name: Configure etcd - hosts: oo_etcd_to_config - any_errors_fatal: true - roles: - - role: os_firewall - - role: openshift_etcd - etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - - role: nickhammond.logrotate - -- name: etcd Install Checkpoint End - hosts: all - gather_facts: false - tasks: - - name: Set etcd install 'Complete' - run_once: true - set_stats: - data: - installer_phase_etcd: - status: "Complete" - end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-etcd/embedded2external.yml b/playbooks/common/openshift-etcd/embedded2external.yml deleted file mode 100644 index b16b78c4f..000000000 --- a/playbooks/common/openshift-etcd/embedded2external.yml +++ /dev/null @@ -1,172 +0,0 @@ ---- -- name: Pre-migrate checks - hosts: localhost - tasks: - # Check there is only one etcd host - - assert: - that: groups.oo_etcd_to_config | default([]) | length == 1 - msg: "[etcd] group must contain only one host" - # Check there is only one master - - assert: - that: groups.oo_masters_to_config | default([]) | length == 1 - msg: "[master] group must contain only one host" - -# 1. stop a master -- name: Prepare masters for etcd data migration - hosts: oo_first_master - roles: - - role: openshift_facts - tasks: - - name: Check the master API is ready - include_role: - name: openshift_master - tasks_from: check_master_api_is_ready - - set_fact: - master_service: "{{ openshift.common.service_type + '-master' }}" - embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - - debug: - msg: "master service name: {{ master_service }}" - - name: Stop master - service: - name: "{{ master_service }}" - state: stopped - # 2. backup embedded etcd - # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285 - - include_role: - name: etcd - tasks_from: backup - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_embedded_etcd: "{{ true }}" - r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" - - - include_role: - name: etcd - tasks_from: backup.archive - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_embedded_etcd: "{{ true }}" - r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" - -# 3. deploy certificates (for etcd and master) -- include: ca.yml - -- include: server_certificates.yml - -- name: Backup etcd client certificates for master host - hosts: oo_first_master - tasks: - - include_role: - name: etcd - tasks_from: backup_master_etcd_certificates - -- name: Redeploy master etcd certificates - include: master_etcd_certificates.yml - vars: - etcd_certificates_redeploy: "{{ true }}" - -# 4. deploy external etcd -- include: ../openshift-etcd/config.yml - -# 5. stop external etcd -- name: Cleanse etcd - hosts: oo_etcd_to_config[0] - gather_facts: no - pre_tasks: - - include_role: - name: etcd - tasks_from: disable_etcd - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - - include_role: - name: etcd - tasks_from: clean_data - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - -# 6. copy the embedded etcd backup to the external host -# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory -- name: Copy embedded etcd backup to the external host - hosts: localhost - tasks: - - name: Create local temp directory for syncing etcd backup - local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX - register: g_etcd_client_mktemp - changed_when: False - become: no - - - include_role: - name: etcd - tasks_from: backup.fetch - vars: - r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_first_master.0].openshift.common.etcd_runtime }}" - etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_embedded_etcd: "{{ true }}" - r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - delegate_to: "{{ groups.oo_first_master[0] }}" - - - include_role: - name: etcd - tasks_from: backup.copy - vars: - r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.etcd_runtime }}" - etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - delegate_to: "{{ groups.oo_etcd_to_config[0] }}" - - - debug: - msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}" - - - name: Delete temporary directory - local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent - changed_when: False - become: no - -# 7. force new cluster from the backup -- name: Force new etcd cluster - hosts: oo_etcd_to_config[0] - tasks: - - include_role: - name: etcd - tasks_from: backup.unarchive - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - - - include_role: - name: etcd - tasks_from: backup.force_new_cluster - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - r_etcd_common_backup_tag: pre-migrate - r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - etcd_peer: "{{ openshift.common.ip }}" - etcd_url_scheme: "https" - etcd_peer_url_scheme: "https" - -# 8. re-configure master to use the external etcd -- name: Configure master to use external etcd - hosts: oo_first_master - tasks: - - include_role: - name: openshift_master - tasks_from: configure_external_etcd - vars: - etcd_peer_url_scheme: "https" - etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}" - etcd_peer_port: 2379 - - # 9. start the master - - name: Start master - service: - name: "{{ master_service }}" - state: started - register: service_status - until: service_status.state is defined and service_status.state == "started" - retries: 5 - delay: 10 diff --git a/playbooks/common/openshift-etcd/filter_plugins b/playbooks/common/openshift-etcd/filter_plugins deleted file mode 120000 index 99a95e4ca..000000000 --- a/playbooks/common/openshift-etcd/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../filter_plugins \ No newline at end of file diff --git a/playbooks/common/openshift-etcd/lookup_plugins b/playbooks/common/openshift-etcd/lookup_plugins deleted file mode 120000 index ac79701db..000000000 --- a/playbooks/common/openshift-etcd/lookup_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../lookup_plugins \ No newline at end of file diff --git a/playbooks/common/openshift-etcd/master_etcd_certificates.yml b/playbooks/common/openshift-etcd/master_etcd_certificates.yml deleted file mode 100644 index 0a25aac57..000000000 --- a/playbooks/common/openshift-etcd/master_etcd_certificates.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Create etcd client certificates for master hosts - hosts: oo_masters_to_config - any_errors_fatal: true - roles: - - role: openshift_etcd_facts - - role: openshift_etcd_client_certificates - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" - etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" - etcd_cert_prefix: "master.etcd-" - openshift_ca_host: "{{ groups.oo_first_master.0 }}" - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config diff --git a/playbooks/common/openshift-etcd/migrate.yml b/playbooks/common/openshift-etcd/migrate.yml deleted file mode 100644 index 31362f2f6..000000000 --- a/playbooks/common/openshift-etcd/migrate.yml +++ /dev/null @@ -1,169 +0,0 @@ ---- -- name: Check if the master has embedded etcd - hosts: localhost - connection: local - become: no - gather_facts: no - tags: - - always - tasks: - - fail: - msg: "Migration of an embedded etcd is not supported. Please, migrate the embedded etcd into an external etcd first." - when: - - groups.oo_etcd_to_config | default([]) | length == 0 - -- name: Run pre-checks - hosts: oo_etcd_to_migrate - tasks: - - include_role: - name: etcd - tasks_from: migrate.pre_check - vars: - r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" - etcd_peer: "{{ ansible_default_ipv4.address }}" - -# TODO: This will be different for release-3.6 branch -- name: Prepare masters for etcd data migration - hosts: oo_masters_to_config - tasks: - - set_fact: - master_services: - - "{{ openshift.common.service_type + '-master-controllers' }}" - - "{{ openshift.common.service_type + '-master-api' }}" - - debug: - msg: "master service name: {{ master_services }}" - - name: Stop masters - service: - name: "{{ item }}" - state: stopped - with_items: "{{ master_services }}" - -- name: Backup v2 data - hosts: oo_etcd_to_migrate - gather_facts: no - roles: - - role: openshift_facts - post_tasks: - - include_role: - name: etcd - tasks_from: backup - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - r_etcd_common_backup_tag: pre-migration - r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" - r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - -- name: Gate on etcd backup - hosts: localhost - connection: local - become: no - tasks: - - set_fact: - etcd_backup_completed: "{{ hostvars - | oo_select_keys(groups.oo_etcd_to_migrate) - | oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}" - - set_fact: - etcd_backup_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_backup_completed) | list }}" - - fail: - msg: "Migration cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" - when: - - etcd_backup_failed | length > 0 - -- name: Stop etcd - hosts: oo_etcd_to_migrate - gather_facts: no - pre_tasks: - - include_role: - name: etcd - tasks_from: disable_etcd - vars: - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - -- name: Migrate data on first etcd - hosts: oo_etcd_to_migrate[0] - gather_facts: no - tasks: - - include_role: - name: etcd - tasks_from: migrate - vars: - r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" - etcd_peer: "{{ openshift.common.ip }}" - etcd_url_scheme: "https" - etcd_peer_url_scheme: "https" - -- name: Clean data stores on remaining etcd hosts - hosts: oo_etcd_to_migrate[1:] - gather_facts: no - tasks: - - include_role: - name: etcd - tasks_from: clean_data - vars: - r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" - etcd_peer: "{{ openshift.common.ip }}" - etcd_url_scheme: "https" - etcd_peer_url_scheme: "https" - - name: Add etcd hosts - delegate_to: localhost - add_host: - name: "{{ item }}" - groups: oo_new_etcd_to_config - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_become: "{{ g_sudo | default(omit) }}" - with_items: "{{ groups.oo_etcd_to_migrate[1:] | default([]) }}" - changed_when: no - - name: Set success - set_fact: - r_etcd_migrate_success: true - -- include: ./scaleup.yml - -- name: Gate on etcd migration - hosts: oo_masters_to_config - gather_facts: no - tasks: - - set_fact: - etcd_migration_completed: "{{ hostvars - | oo_select_keys(groups.oo_etcd_to_migrate) - | oo_collect('inventory_hostname', {'r_etcd_migrate_success': true}) }}" - - set_fact: - etcd_migration_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_migration_completed) | list }}" - -- name: Add TTLs on the first master - hosts: oo_first_master[0] - tasks: - - include_role: - name: etcd - tasks_from: migrate.add_ttls - vars: - etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}" - etcd_url_scheme: "https" - etcd_peer_url_scheme: "https" - when: etcd_migration_failed | length == 0 - -- name: Configure masters if etcd data migration is succesfull - hosts: oo_masters_to_config - tasks: - - include_role: - name: etcd - tasks_from: migrate.configure_master - when: etcd_migration_failed | length == 0 - - debug: - msg: "Skipping master re-configuration since migration failed." - when: - - etcd_migration_failed | length > 0 - - name: Start master services - service: - name: "{{ item }}" - state: started - register: service_status - # Sometimes the master-api, resp. master-controllers fails to start for the first time - until: service_status.state is defined and service_status.state == "started" - retries: 5 - delay: 10 - with_items: "{{ master_services[::-1] }}" - - fail: - msg: "Migration failed. The following hosts were not properly migrated: {{ etcd_migration_failed | join(',') }}" - when: - - etcd_migration_failed | length > 0 diff --git a/playbooks/common/openshift-etcd/restart.yml b/playbooks/common/openshift-etcd/restart.yml deleted file mode 100644 index 5eaea5ae8..000000000 --- a/playbooks/common/openshift-etcd/restart.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Restart etcd - hosts: oo_etcd_to_config - serial: 1 - tasks: - - name: restart etcd - service: - name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" - state: restarted - when: - - not g_etcd_certificates_expired | default(false) | bool - -- name: Restart etcd - hosts: oo_etcd_to_config - tasks: - - name: stop etcd - service: - name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" - state: stopped - when: - - g_etcd_certificates_expired | default(false) | bool - - name: start etcd - service: - name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" - state: started - when: - - g_etcd_certificates_expired | default(false) | bool diff --git a/playbooks/common/openshift-etcd/roles b/playbooks/common/openshift-etcd/roles deleted file mode 120000 index e2b799b9d..000000000 --- a/playbooks/common/openshift-etcd/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles/ \ No newline at end of file diff --git a/playbooks/common/openshift-etcd/scaleup.yml b/playbooks/common/openshift-etcd/scaleup.yml deleted file mode 100644 index 20061366c..000000000 --- a/playbooks/common/openshift-etcd/scaleup.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -- name: Gather facts - hosts: oo_etcd_to_config:oo_new_etcd_to_config - roles: - - openshift_etcd_facts - post_tasks: - - set_fact: - etcd_hostname: "{{ etcd_hostname }}" - etcd_ip: "{{ etcd_ip }}" - -- name: Configure etcd - hosts: oo_new_etcd_to_config - serial: 1 - any_errors_fatal: true - vars: - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - pre_tasks: - - name: Add new etcd members to cluster - command: > - /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} - --key-file {{ etcd_peer_key_file }} - --ca-file {{ etcd_peer_ca_file }} - -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }} - member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }} - delegate_to: "{{ etcd_ca_host }}" - failed_when: - - etcd_add_check.rc == 1 - - ("peerURL exists" not in etcd_add_check.stderr) - register: etcd_add_check - retries: 3 - delay: 10 - until: etcd_add_check.rc == 0 - - include_role: - name: etcd - tasks_from: server_certificates - vars: - etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - roles: - - role: os_firewall - when: etcd_add_check.rc == 0 - - role: openshift_etcd - when: etcd_add_check.rc == 0 - etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}" - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_initial_cluster_state: "existing" - etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" - etcd_ca_setup: False - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - - role: nickhammond.logrotate - when: etcd_add_check.rc == 0 - post_tasks: - - name: Verify cluster is stable - command: > - /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} - --key-file {{ etcd_peer_key_file }} - --ca-file {{ etcd_peer_ca_file }} - -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }} - cluster-health - register: scaleup_health - retries: 3 - delay: 30 - until: scaleup_health.rc == 0 - -- name: Update master etcd client urls - hosts: oo_masters_to_config - serial: 1 - vars: - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - openshift_ca_host: "{{ groups.oo_first_master.0 }}" - openshift_master_etcd_hosts: "{{ hostvars - | oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) )) - | oo_collect('openshift.common.hostname') - | default(none, true) }}" - openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}" - roles: - - role: openshift_master_facts - post_tasks: - - include_role: - name: openshift_master - tasks_from: update_etcd_client_urls diff --git a/playbooks/common/openshift-etcd/server_certificates.yml b/playbooks/common/openshift-etcd/server_certificates.yml deleted file mode 100644 index 10e06747b..000000000 --- a/playbooks/common/openshift-etcd/server_certificates.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Create etcd server certificates for etcd hosts - hosts: oo_etcd_to_config - any_errors_fatal: true - roles: - - role: openshift_etcd_facts - post_tasks: - - include_role: - name: etcd - tasks_from: server_certificates - vars: - etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" - etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" - etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" - r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" diff --git a/playbooks/common/openshift-master/scaleup.yml b/playbooks/common/openshift-master/scaleup.yml index 4c415ebce..cc0196e5b 100644 --- a/playbooks/common/openshift-master/scaleup.yml +++ b/playbooks/common/openshift-master/scaleup.yml @@ -46,7 +46,7 @@ - include: ../openshift-master/set_network_facts.yml -- include: ../openshift-etcd/certificates.yml +- include: ../../openshift-etcd/private/certificates.yml - include: ../openshift-master/config.yml diff --git a/playbooks/openshift-etcd/certificates.yml b/playbooks/openshift-etcd/certificates.yml new file mode 100644 index 000000000..8240d3fe6 --- /dev/null +++ b/playbooks/openshift-etcd/certificates.yml @@ -0,0 +1,6 @@ +--- +- include: ../init/main.yml + +- include: private/ca.yml + +- include: private/certificates.yml diff --git a/playbooks/openshift-etcd/config.yml b/playbooks/openshift-etcd/config.yml new file mode 100644 index 000000000..8ee57ce8d --- /dev/null +++ b/playbooks/openshift-etcd/config.yml @@ -0,0 +1,4 @@ +--- +- include: ../init/main.yml + +- include: private/config.yml diff --git a/playbooks/openshift-etcd/embedded2external.yml b/playbooks/openshift-etcd/embedded2external.yml new file mode 100644 index 000000000..a11b140de --- /dev/null +++ b/playbooks/openshift-etcd/embedded2external.yml @@ -0,0 +1,4 @@ +--- +- include: ../init/main.yml + +- include: private/embedded2external.yml diff --git a/playbooks/openshift-etcd/migrate.yml b/playbooks/openshift-etcd/migrate.yml new file mode 100644 index 000000000..a307c2740 --- /dev/null +++ b/playbooks/openshift-etcd/migrate.yml @@ -0,0 +1,4 @@ +--- +- include: ../init/main.yml + +- include: private/migrate.yml diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml new file mode 100644 index 000000000..ac5543be9 --- /dev/null +++ b/playbooks/openshift-etcd/private/ca.yml @@ -0,0 +1,15 @@ +--- +- name: Generate new etcd CA + hosts: oo_first_etcd + roles: + - role: openshift_etcd_facts + tasks: + - include_role: + name: etcd + tasks_from: ca + vars: + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + when: + - etcd_ca_setup | default(True) | bool diff --git a/playbooks/openshift-etcd/private/certificates.yml b/playbooks/openshift-etcd/private/certificates.yml new file mode 100644 index 000000000..eb6b94f33 --- /dev/null +++ b/playbooks/openshift-etcd/private/certificates.yml @@ -0,0 +1,4 @@ +--- +- include: server_certificates.yml + +- include: master_etcd_certificates.yml diff --git a/playbooks/openshift-etcd/private/config.yml b/playbooks/openshift-etcd/private/config.yml new file mode 100644 index 000000000..3fe483785 --- /dev/null +++ b/playbooks/openshift-etcd/private/config.yml @@ -0,0 +1,40 @@ +--- +- name: etcd Install Checkpoint Start + hosts: all + gather_facts: false + tasks: + - name: Set etcd install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_etcd: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + +- include: ca.yml + +- include: certificates.yml + +- name: Configure etcd + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - role: os_firewall + - role: openshift_etcd + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + - role: nickhammond.logrotate + +- name: etcd Install Checkpoint End + hosts: all + gather_facts: false + tasks: + - name: Set etcd install 'Complete' + run_once: true + set_stats: + data: + installer_phase_etcd: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml new file mode 100644 index 000000000..0bd9912b0 --- /dev/null +++ b/playbooks/openshift-etcd/private/embedded2external.yml @@ -0,0 +1,172 @@ +--- +- name: Pre-migrate checks + hosts: localhost + tasks: + # Check there is only one etcd host + - assert: + that: groups.oo_etcd_to_config | default([]) | length == 1 + msg: "[etcd] group must contain only one host" + # Check there is only one master + - assert: + that: groups.oo_masters_to_config | default([]) | length == 1 + msg: "[master] group must contain only one host" + +# 1. stop a master +- name: Prepare masters for etcd data migration + hosts: oo_first_master + roles: + - role: openshift_facts + tasks: + - name: Check the master API is ready + include_role: + name: openshift_master + tasks_from: check_master_api_is_ready + - set_fact: + master_service: "{{ openshift.common.service_type + '-master' }}" + embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + - debug: + msg: "master service name: {{ master_service }}" + - name: Stop master + service: + name: "{{ master_service }}" + state: stopped + # 2. backup embedded etcd + # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285 + - include_role: + name: etcd + tasks_from: backup + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" + + - include_role: + name: etcd + tasks_from: backup.archive + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}" + +# 3. deploy certificates (for etcd and master) +- include: ca.yml + +- include: server_certificates.yml + +- name: Backup etcd client certificates for master host + hosts: oo_first_master + tasks: + - include_role: + name: etcd + tasks_from: backup_master_etcd_certificates + +- name: Redeploy master etcd certificates + include: master_etcd_certificates.yml + vars: + etcd_certificates_redeploy: "{{ true }}" + +# 4. deploy external etcd +- include: config.yml + +# 5. stop external etcd +- name: Cleanse etcd + hosts: oo_etcd_to_config[0] + gather_facts: no + pre_tasks: + - include_role: + name: etcd + tasks_from: disable_etcd + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + - include_role: + name: etcd + tasks_from: clean_data + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + +# 6. copy the embedded etcd backup to the external host +# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory +- name: Copy embedded etcd backup to the external host + hosts: localhost + tasks: + - name: Create local temp directory for syncing etcd backup + local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX + register: g_etcd_client_mktemp + changed_when: False + become: no + + - include_role: + name: etcd + tasks_from: backup.fetch + vars: + r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_first_master.0].openshift.common.etcd_runtime }}" + etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_embedded_etcd: "{{ true }}" + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + delegate_to: "{{ groups.oo_first_master[0] }}" + + - include_role: + name: etcd + tasks_from: backup.copy + vars: + r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.etcd_runtime }}" + etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + delegate_to: "{{ groups.oo_etcd_to_config[0] }}" + + - debug: + msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}" + + - name: Delete temporary directory + local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent + changed_when: False + become: no + +# 7. force new cluster from the backup +- name: Force new etcd cluster + hosts: oo_etcd_to_config[0] + tasks: + - include_role: + name: etcd + tasks_from: backup.unarchive + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + + - include_role: + name: etcd + tasks_from: backup.force_new_cluster + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + r_etcd_common_backup_tag: pre-migrate + r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + +# 8. re-configure master to use the external etcd +- name: Configure master to use external etcd + hosts: oo_first_master + tasks: + - include_role: + name: openshift_master + tasks_from: configure_external_etcd + vars: + etcd_peer_url_scheme: "https" + etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}" + etcd_peer_port: 2379 + + # 9. start the master + - name: Start master + service: + name: "{{ master_service }}" + state: started + register: service_status + until: service_status.state is defined and service_status.state == "started" + retries: 5 + delay: 10 diff --git a/playbooks/openshift-etcd/private/filter_plugins b/playbooks/openshift-etcd/private/filter_plugins new file mode 120000 index 000000000..99a95e4ca --- /dev/null +++ b/playbooks/openshift-etcd/private/filter_plugins @@ -0,0 +1 @@ +../../../filter_plugins \ No newline at end of file diff --git a/playbooks/openshift-etcd/private/lookup_plugins b/playbooks/openshift-etcd/private/lookup_plugins new file mode 120000 index 000000000..ac79701db --- /dev/null +++ b/playbooks/openshift-etcd/private/lookup_plugins @@ -0,0 +1 @@ +../../../lookup_plugins \ No newline at end of file diff --git a/playbooks/openshift-etcd/private/master_etcd_certificates.yml b/playbooks/openshift-etcd/private/master_etcd_certificates.yml new file mode 100644 index 000000000..0a25aac57 --- /dev/null +++ b/playbooks/openshift-etcd/private/master_etcd_certificates.yml @@ -0,0 +1,14 @@ +--- +- name: Create etcd client certificates for master hosts + hosts: oo_masters_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_facts + - role: openshift_etcd_client_certificates + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}" + etcd_cert_config_dir: "{{ openshift.common.config_base }}/master" + etcd_cert_prefix: "master.etcd-" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml new file mode 100644 index 000000000..31362f2f6 --- /dev/null +++ b/playbooks/openshift-etcd/private/migrate.yml @@ -0,0 +1,169 @@ +--- +- name: Check if the master has embedded etcd + hosts: localhost + connection: local + become: no + gather_facts: no + tags: + - always + tasks: + - fail: + msg: "Migration of an embedded etcd is not supported. Please, migrate the embedded etcd into an external etcd first." + when: + - groups.oo_etcd_to_config | default([]) | length == 0 + +- name: Run pre-checks + hosts: oo_etcd_to_migrate + tasks: + - include_role: + name: etcd + tasks_from: migrate.pre_check + vars: + r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + etcd_peer: "{{ ansible_default_ipv4.address }}" + +# TODO: This will be different for release-3.6 branch +- name: Prepare masters for etcd data migration + hosts: oo_masters_to_config + tasks: + - set_fact: + master_services: + - "{{ openshift.common.service_type + '-master-controllers' }}" + - "{{ openshift.common.service_type + '-master-api' }}" + - debug: + msg: "master service name: {{ master_services }}" + - name: Stop masters + service: + name: "{{ item }}" + state: stopped + with_items: "{{ master_services }}" + +- name: Backup v2 data + hosts: oo_etcd_to_migrate + gather_facts: no + roles: + - role: openshift_facts + post_tasks: + - include_role: + name: etcd + tasks_from: backup + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + r_etcd_common_backup_tag: pre-migration + r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | oo_select_keys(groups.oo_etcd_to_migrate) + | oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_backup_completed) | list }}" + - fail: + msg: "Migration cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: + - etcd_backup_failed | length > 0 + +- name: Stop etcd + hosts: oo_etcd_to_migrate + gather_facts: no + pre_tasks: + - include_role: + name: etcd + tasks_from: disable_etcd + vars: + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + +- name: Migrate data on first etcd + hosts: oo_etcd_to_migrate[0] + gather_facts: no + tasks: + - include_role: + name: etcd + tasks_from: migrate + vars: + r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + +- name: Clean data stores on remaining etcd hosts + hosts: oo_etcd_to_migrate[1:] + gather_facts: no + tasks: + - include_role: + name: etcd + tasks_from: clean_data + vars: + r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + etcd_peer: "{{ openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + - name: Add etcd hosts + delegate_to: localhost + add_host: + name: "{{ item }}" + groups: oo_new_etcd_to_config + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_become: "{{ g_sudo | default(omit) }}" + with_items: "{{ groups.oo_etcd_to_migrate[1:] | default([]) }}" + changed_when: no + - name: Set success + set_fact: + r_etcd_migrate_success: true + +- include: ./scaleup.yml + +- name: Gate on etcd migration + hosts: oo_masters_to_config + gather_facts: no + tasks: + - set_fact: + etcd_migration_completed: "{{ hostvars + | oo_select_keys(groups.oo_etcd_to_migrate) + | oo_collect('inventory_hostname', {'r_etcd_migrate_success': true}) }}" + - set_fact: + etcd_migration_failed: "{{ groups.oo_etcd_to_migrate | difference(etcd_migration_completed) | list }}" + +- name: Add TTLs on the first master + hosts: oo_first_master[0] + tasks: + - include_role: + name: etcd + tasks_from: migrate.add_ttls + vars: + etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}" + etcd_url_scheme: "https" + etcd_peer_url_scheme: "https" + when: etcd_migration_failed | length == 0 + +- name: Configure masters if etcd data migration is succesfull + hosts: oo_masters_to_config + tasks: + - include_role: + name: etcd + tasks_from: migrate.configure_master + when: etcd_migration_failed | length == 0 + - debug: + msg: "Skipping master re-configuration since migration failed." + when: + - etcd_migration_failed | length > 0 + - name: Start master services + service: + name: "{{ item }}" + state: started + register: service_status + # Sometimes the master-api, resp. master-controllers fails to start for the first time + until: service_status.state is defined and service_status.state == "started" + retries: 5 + delay: 10 + with_items: "{{ master_services[::-1] }}" + - fail: + msg: "Migration failed. The following hosts were not properly migrated: {{ etcd_migration_failed | join(',') }}" + when: + - etcd_migration_failed | length > 0 diff --git a/playbooks/openshift-etcd/private/restart.yml b/playbooks/openshift-etcd/private/restart.yml new file mode 100644 index 000000000..5eaea5ae8 --- /dev/null +++ b/playbooks/openshift-etcd/private/restart.yml @@ -0,0 +1,27 @@ +--- +- name: Restart etcd + hosts: oo_etcd_to_config + serial: 1 + tasks: + - name: restart etcd + service: + name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" + state: restarted + when: + - not g_etcd_certificates_expired | default(false) | bool + +- name: Restart etcd + hosts: oo_etcd_to_config + tasks: + - name: stop etcd + service: + name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" + state: stopped + when: + - g_etcd_certificates_expired | default(false) | bool + - name: start etcd + service: + name: "{{ 'etcd_container' if openshift.common.etcd_runtime == 'docker' else 'etcd' }}" + state: started + when: + - g_etcd_certificates_expired | default(false) | bool diff --git a/playbooks/openshift-etcd/private/roles b/playbooks/openshift-etcd/private/roles new file mode 120000 index 000000000..20c4c58cf --- /dev/null +++ b/playbooks/openshift-etcd/private/roles @@ -0,0 +1 @@ +../../../roles \ No newline at end of file diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml new file mode 100644 index 000000000..20061366c --- /dev/null +++ b/playbooks/openshift-etcd/private/scaleup.yml @@ -0,0 +1,83 @@ +--- +- name: Gather facts + hosts: oo_etcd_to_config:oo_new_etcd_to_config + roles: + - openshift_etcd_facts + post_tasks: + - set_fact: + etcd_hostname: "{{ etcd_hostname }}" + etcd_ip: "{{ etcd_ip }}" + +- name: Configure etcd + hosts: oo_new_etcd_to_config + serial: 1 + any_errors_fatal: true + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + pre_tasks: + - name: Add new etcd members to cluster + command: > + /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} + --key-file {{ etcd_peer_key_file }} + --ca-file {{ etcd_peer_ca_file }} + -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_ip }}:{{ etcd_client_port }} + member add {{ etcd_hostname }} {{ etcd_peer_url_scheme }}://{{ etcd_ip }}:{{ etcd_peer_port }} + delegate_to: "{{ etcd_ca_host }}" + failed_when: + - etcd_add_check.rc == 1 + - ("peerURL exists" not in etcd_add_check.stderr) + register: etcd_add_check + retries: 3 + delay: 10 + until: etcd_add_check.rc == 0 + - include_role: + name: etcd + tasks_from: server_certificates + vars: + etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + roles: + - role: os_firewall + when: etcd_add_check.rc == 0 + - role: openshift_etcd + when: etcd_add_check.rc == 0 + etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}" + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_initial_cluster_state: "existing" + etcd_initial_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" + etcd_ca_setup: False + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" + - role: nickhammond.logrotate + when: etcd_add_check.rc == 0 + post_tasks: + - name: Verify cluster is stable + command: > + /usr/bin/etcdctl --cert-file {{ etcd_peer_cert_file }} + --key-file {{ etcd_peer_key_file }} + --ca-file {{ etcd_peer_ca_file }} + -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }} + cluster-health + register: scaleup_health + retries: 3 + delay: 30 + until: scaleup_health.rc == 0 + +- name: Update master etcd client urls + hosts: oo_masters_to_config + serial: 1 + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + openshift_ca_host: "{{ groups.oo_first_master.0 }}" + openshift_master_etcd_hosts: "{{ hostvars + | oo_select_keys(groups['oo_etcd_to_config'] | union(groups['oo_new_etcd_to_config'] | default([]) )) + | oo_collect('openshift.common.hostname') + | default(none, true) }}" + openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}" + roles: + - role: openshift_master_facts + post_tasks: + - include_role: + name: openshift_master + tasks_from: update_etcd_client_urls diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml new file mode 100644 index 000000000..10e06747b --- /dev/null +++ b/playbooks/openshift-etcd/private/server_certificates.yml @@ -0,0 +1,15 @@ +--- +- name: Create etcd server certificates for etcd hosts + hosts: oo_etcd_to_config + any_errors_fatal: true + roles: + - role: openshift_etcd_facts + post_tasks: + - include_role: + name: etcd + tasks_from: server_certificates + vars: + etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" + etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" + etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" + r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" diff --git a/playbooks/openshift-etcd/restart.yml b/playbooks/openshift-etcd/restart.yml new file mode 100644 index 000000000..5e28e274e --- /dev/null +++ b/playbooks/openshift-etcd/restart.yml @@ -0,0 +1,4 @@ +--- +- include: ../init/main.yml + +- include: private/restart.yml diff --git a/playbooks/openshift-etcd/scaleup.yml b/playbooks/openshift-etcd/scaleup.yml new file mode 100644 index 000000000..d83697131 --- /dev/null +++ b/playbooks/openshift-etcd/scaleup.yml @@ -0,0 +1,4 @@ +--- +- include: ../init/main.yml + +- include: private/scaleup.yml -- cgit v1.2.3