From 7197aba51d24ab2cf6cde77efa853903d7ddd5ba Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 28 Oct 2015 12:12:39 -0300 Subject: Block upgrade if targetting enterprise deployment type. enterprise is being phased out in favor of openshift-enterprise, you need to specify where you wish to go. --- playbooks/adhoc/upgrades/upgrade.yml | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 56a1df860..e8d6a335d 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,4 +1,12 @@ --- +- name: Verify deployment type + hosts: masters + tasks: + # Checking the global deployment type rather than host facts, this is about + # what the user is requesting. + - fail: msg="Deployment type enterprise not supported for upgrade" + when: deployment_type == "enterprise" + - name: Re-Run cluster configuration to apply latest configuration changes include: ../../common/openshift-cluster/config.yml vars: -- cgit v1.2.3 From 3d7c5c6fd545112d87fa09e4a8c3f3cbc1cda1ee Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 28 Oct 2015 15:29:52 -0300 Subject: First cut at checking available disk space for etcd backup. --- playbooks/adhoc/upgrades/upgrade.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index e8d6a335d..c2c1d57e8 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,5 +1,5 @@ --- -- name: Verify deployment type +- name: Verify upgrade can proceed hosts: masters tasks: # Checking the global deployment type rather than host facts, this is about @@ -7,6 +7,31 @@ - fail: msg="Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" +- name: Backup etcd + hosts: masters + vars: + embedded_etcd: "{{ openshift.master.embedded_etcd }}" + roles: + - openshift_facts + tasks: + - name: display all variables set for the current host + debug: + var: hostvars[inventory_hostname] + - debug: var=embedded_etcd + - name: Check available data dir disk space + shell: > + df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 + register: avail_disk + when: embedded_etcd | bool + - debug: var=avail_disk.stdout + - name: Check current etcd disk usage + shell: > + df --output=avail -k /var/lib/openshift/openshift.local.etcd/ | tail -n 1 + register: etc_disk_usage + when: embedded_etcd | bool + - debug: var=etc_disk_usage.stdout + - fail: msg="All done for now." + - name: Re-Run cluster configuration to apply latest configuration changes include: ../../common/openshift-cluster/config.yml vars: -- cgit v1.2.3 From 18df191a381ced63613dea302c5bf2e172d3c803 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 29 Oct 2015 15:07:42 -0300 Subject: Functional disk space checking for etcd backup. --- playbooks/adhoc/upgrades/upgrade.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index c2c1d57e8..ec0eca66c 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -26,10 +26,13 @@ - debug: var=avail_disk.stdout - name: Check current etcd disk usage shell: > - df --output=avail -k /var/lib/openshift/openshift.local.etcd/ | tail -n 1 + du -k {{ openshift.common.data_dir }}/openshift.local.etcd | tail -n 1 | cut -f1 register: etc_disk_usage when: embedded_etcd | bool - debug: var=etc_disk_usage.stdout + - name: Abort if insufficient disk space for etcd backup + fail: msg="{{ etc_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available." + when: (embedded_etcd | bool) and (etc_disk_usage.stdout|int > avail_disk.stdout|int) - fail: msg="All done for now." - name: Re-Run cluster configuration to apply latest configuration changes -- cgit v1.2.3 From ef2e4fc1fa247ba51394766f9cf9279f0beaf0b0 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 30 Oct 2015 09:02:26 -0300 Subject: Add etcd_data_dir fact. --- playbooks/adhoc/upgrades/upgrade.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index ec0eca66c..948a320a2 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -14,25 +14,26 @@ roles: - openshift_facts tasks: - - name: display all variables set for the current host - debug: - var: hostvars[inventory_hostname] - - debug: var=embedded_etcd - - name: Check available data dir disk space + - debug: var=etcd_data_dir + - name: Check available disk space for etcd backup + # We assume to be using the data dir for all backups. shell: > df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 register: avail_disk - when: embedded_etcd | bool - - debug: var=avail_disk.stdout - - name: Check current etcd disk usage + + - name: Check current embedded etcd disk usage shell: > - du -k {{ openshift.common.data_dir }}/openshift.local.etcd | tail -n 1 | cut -f1 - register: etc_disk_usage + du -k {{ openshift.master.etcd_data_dir }} | tail -n 1 | cut -f1 + register: etcd_disk_usage when: embedded_etcd | bool - - debug: var=etc_disk_usage.stdout + - name: Abort if insufficient disk space for etcd backup - fail: msg="{{ etc_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available." - when: (embedded_etcd | bool) and (etc_disk_usage.stdout|int > avail_disk.stdout|int) + fail: msg="{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available." + when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) + - name: Install etcd (for etcdctl) + yum: pkg=etcd state=latest + - name: Generate etcd backup + command: etcdctl backup - fail: msg="All done for now." - name: Re-Run cluster configuration to apply latest configuration changes -- cgit v1.2.3 From 561c347d54c3a124f64b54eedbf6d2d56fb07c34 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 30 Oct 2015 09:41:12 -0300 Subject: Generate timestamped etcd backups. --- playbooks/adhoc/upgrades/upgrade.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 948a320a2..11d89a3da 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -11,10 +11,10 @@ hosts: masters vars: embedded_etcd: "{{ openshift.master.embedded_etcd }}" + timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" roles: - openshift_facts tasks: - - debug: var=etcd_data_dir - name: Check available disk space for etcd backup # We assume to be using the data dir for all backups. shell: > @@ -33,7 +33,7 @@ - name: Install etcd (for etcdctl) yum: pkg=etcd state=latest - name: Generate etcd backup - command: etcdctl backup + command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} - fail: msg="All done for now." - name: Re-Run cluster configuration to apply latest configuration changes -- cgit v1.2.3 From 83e20a43c7b7e6876a7828d02b7fc96b8c8bbf38 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 30 Oct 2015 11:20:44 -0300 Subject: Remove devel fail and let upgrade proceed. --- playbooks/adhoc/upgrades/upgrade.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 11d89a3da..30896dc8e 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -34,7 +34,6 @@ yum: pkg=etcd state=latest - name: Generate etcd backup command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} - - fail: msg="All done for now." - name: Re-Run cluster configuration to apply latest configuration changes include: ../../common/openshift-cluster/config.yml @@ -50,6 +49,8 @@ vars: openshift_version: "{{ openshift_pkg_version | default('') }}" tasks: + - name: Upgrade to latest available kernel + yum: pkg=kernel state=latest - name: Upgrade master packages yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest - name: Restart master services -- cgit v1.2.3 From 866dcbe962464a571f24e1e4ae5a9e928f7bbc47 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Mon, 2 Nov 2015 13:14:26 -0400 Subject: Fix etcd backup bug with not-yet-created /var/lib/origin symlink --- playbooks/adhoc/upgrades/upgrade.yml | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 30896dc8e..c543a4416 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -15,6 +15,11 @@ roles: - openshift_facts tasks: + - stat: path=/var/lib/openshift + register: var_lib_openshift + - name: Create origin symlink if necessary + file: src=/var/lib/openshift/ dest=/var/lib/origin state=link + when: var_lib_openshift.stat.exists == True - name: Check available disk space for etcd backup # We assume to be using the data dir for all backups. shell: > -- cgit v1.2.3 From 078a44715b1190f4a7e0b098b0dc87446d88ac8c Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Mon, 2 Nov 2015 17:09:54 -0500 Subject: initial module framework --- .../upgrades/library/openshift_upgrade_config.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 playbooks/adhoc/upgrades/library/openshift_upgrade_config.py (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py new file mode 100755 index 000000000..157fbc5c2 --- /dev/null +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# vim: expandtab:tabstop=4:shiftwidth=4 + +"""Ansible module for modifying OpenShift configs during an upgrade""" + +DOCUMENTATION = ''' +--- +module: openshift_upgrade_config +short_description: OpenShift Upgrade Config +author: Jason DeTiberus +requirements: [ ] +''' +EXAMPLES = ''' +''' + +def upgrade_master_3_0_to_3_1(backup): + pass + + +def upgrade_master(from_version, to_version, backup): + if from_version == '3.0': + if to_version == '3.1': + upgrade_master_3_0_to_3_1(backup) + + +def main(): + """ main """ + # disabling pylint errors for global-variable-undefined and invalid-name + # for 'global module' usage, since it is required to use ansible_facts + # pylint: disable=global-variable-undefined, invalid-name + global module + + module = AnsibleModule( + argument_spec=dict( + from_version=dict(required=True, choices=['3.0']), + to_version=dict(required=True, choices=['3.1']), + role=dict(required=True, choices=['master']), + backup=dict(required=False, default=True, type='bool') + ), + supports_check_mode=True, + ) + + changed = False + + from_version = module.params['from_version'] + to_version = module.params['to_version'] + role = module.params['role'] + backup = module.params['backup'] + + if role == 'master': + upgrade_master(from_version, to_version, backup) + + return module.exit_json(changed=changed) + +# ignore pylint errors related to the module_utils import +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import +# import module snippets +from ansible.module_utils.basic import * + +if __name__ == '__main__': + main() -- cgit v1.2.3 From cbf98f53b0975a829a5720c33149c7014d5bd7cd Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 3 Nov 2015 09:27:13 -0400 Subject: Implement master-config.yaml upgrade for v1beta3 apiLevel removal. --- .../upgrades/library/openshift_upgrade_config.py | 53 ++++++++++++++++++++-- playbooks/adhoc/upgrades/upgrade.yml | 25 ++++++---- 2 files changed, 66 insertions(+), 12 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 157fbc5c2..f41c6fb59 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -2,6 +2,12 @@ # -*- coding: utf-8 -*- # vim: expandtab:tabstop=4:shiftwidth=4 +import os +import shutil +import yaml + +from datetime import datetime + """Ansible module for modifying OpenShift configs during an upgrade""" DOCUMENTATION = ''' @@ -14,14 +20,53 @@ requirements: [ ] EXAMPLES = ''' ''' +def get_cfg_dir(): + cfg_path = '/etc/origin/' + if not os.path.exists(cfg_path): + cfg_path = '/etc/openshift/' + return cfg_path + def upgrade_master_3_0_to_3_1(backup): - pass + changed = False + + # Facts do not get transferred to the hosts where custom modules run, + # need to make some assumptions here. + master_config = os.path.join(get_cfg_dir(), 'master/master-config.yaml') + + f = open(master_config, 'r') + config = yaml.safe_load(f.read()) + f.close() + + # Remove v1beta3 from apiLevels: + if 'apiLevels' in config and \ + 'v1beta3' in config['apiLevels']: + config['apiLevels'].remove('v1beta3') + changed = True + if 'kubernetesMasterConfig' in config and \ + 'apiLevels' in config['kubernetesMasterConfig'] and \ + 'v1beta3' in config['kubernetesMasterConfig']['apiLevels']: + config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') + changed = True + + if changed: + if backup: + timestamp = datetime.now().strftime('%Y%m%d%H%M%S') + basedir = os.path.split(master_config)[0] + backup_file = os.path.join(basedir, 'master-config.yaml.bak-%s' + % timestamp) + shutil.copyfile(master_config, backup_file) + # Write the modified config: + out_file = open(master_config, 'w') + out_file.write(yaml.safe_dump(config, default_flow_style=False)) + out_file.close() + + return changed def upgrade_master(from_version, to_version, backup): if from_version == '3.0': if to_version == '3.1': - upgrade_master_3_0_to_3_1(backup) + return upgrade_master_3_0_to_3_1(backup) def main(): @@ -41,15 +86,15 @@ def main(): supports_check_mode=True, ) - changed = False from_version = module.params['from_version'] to_version = module.params['to_version'] role = module.params['role'] backup = module.params['backup'] + changed = False if role == 'master': - upgrade_master(from_version, to_version, backup) + changed = upgrade_master(from_version, to_version, backup) return module.exit_json(changed=changed) diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index e55f1536f..6cb6803a9 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -40,6 +40,15 @@ - name: Generate etcd backup command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} +- name: Update 3.0 configuration to 3.1 + hosts: masters + tasks: + tasks: + - debug: var=openshift_master_config_dir + - name: Upgrade configuration. + openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master + - fail: All done for now. + - name: Upgrade base package on masters hosts: masters roles: @@ -50,14 +59,14 @@ - name: Upgrade base package yum: pkg={{ openshift.common.service_type }}{{ openshift_version }} state=latest -- name: Re-Run cluster configuration to apply latest configuration changes - include: ../../common/openshift-cluster/config.yml - vars: - g_etcd_group: "{{ 'etcd' }}" - g_masters_group: "{{ 'masters' }}" - g_nodes_group: "{{ 'nodes' }}" - openshift_cluster_id: "{{ cluster_id | default('default') }}" - openshift_deployment_type: "{{ deployment_type }}" + #- name: Re-Run cluster configuration to apply latest configuration changes + # include: ../../common/openshift-cluster/config.yml + # vars: + # g_etcd_group: "{{ 'etcd' }}" + # g_masters_group: "{{ 'masters' }}" + # g_nodes_group: "{{ 'nodes' }}" + # openshift_cluster_id: "{{ cluster_id | default('default') }}" + # openshift_deployment_type: "{{ deployment_type }}" - name: Upgrade masters hosts: masters -- cgit v1.2.3 From 6b03fcb185c7f37e258dd34ca8b70d5c124cabba Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 3 Nov 2015 11:48:27 -0400 Subject: Add in proxyClientInfo if missing during config upgrade. --- playbooks/adhoc/upgrades/library/openshift_upgrade_config.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index f41c6fb59..95c7b1664 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -26,6 +26,7 @@ def get_cfg_dir(): cfg_path = '/etc/openshift/' return cfg_path + def upgrade_master_3_0_to_3_1(backup): changed = False @@ -42,12 +43,18 @@ def upgrade_master_3_0_to_3_1(backup): 'v1beta3' in config['apiLevels']: config['apiLevels'].remove('v1beta3') changed = True - if 'kubernetesMasterConfig' in config and \ - 'apiLevels' in config['kubernetesMasterConfig'] and \ + if 'apiLevels' in config['kubernetesMasterConfig'] and \ 'v1beta3' in config['kubernetesMasterConfig']['apiLevels']: config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') changed = True + # Add the new master proxy client certs: + if 'proxyClientInfo' not in config['kubernetesMasterConfig']: + config['kubernetesMasterConfig']['proxyClientInfo'] = { + 'certFile': 'master.proxy-client.crt', + 'keyFile': 'master.proxy-client.key' + } + if changed: if backup: timestamp = datetime.now().strftime('%Y%m%d%H%M%S') @@ -86,7 +93,6 @@ def main(): supports_check_mode=True, ) - from_version = module.params['from_version'] to_version = module.params['to_version'] role = module.params['role'] -- cgit v1.2.3 From 74420660b1b12a00392a6d88a257e0cfe1b0a08e Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 3 Nov 2015 12:34:05 -0400 Subject: Move config upgrade to correct place, fix node facts. --- playbooks/adhoc/upgrades/upgrade.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 6cb6803a9..6e7292da1 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -40,15 +40,6 @@ - name: Generate etcd backup command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} -- name: Update 3.0 configuration to 3.1 - hosts: masters - tasks: - tasks: - - debug: var=openshift_master_config_dir - - name: Upgrade configuration. - openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master - - fail: All done for now. - - name: Upgrade base package on masters hosts: masters roles: @@ -77,6 +68,8 @@ yum: pkg=kernel state=latest - name: Upgrade master packages yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest + - name: Upgrade master configuration. + openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master - name: Restart master services service: name="{{ openshift.common.service_type}}-master" state=restarted @@ -84,12 +77,30 @@ hosts: nodes vars: openshift_version: "{{ openshift_pkg_version | default('') }}" + roles: + - openshift_facts tasks: - name: Upgrade node packages yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest - name: Restart node services service: name="{{ openshift.common.service_type }}-node" state=restarted +- name: Evaluate oo_first_master + hosts: localhost + vars: + g_masters_group: "{{ 'masters' }}" + tasks: + - name: display all variables set for the current host + debug: + var: hostvars[inventory_hostname] + - name: Evaluate oo_first_master + add_host: + name: "{{ groups[g_masters_group][0] }}" + groups: oo_first_master + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_sudo: "{{ g_sudo | default(omit) }}" + when: g_masters_group in groups and (groups[g_masters_group] | length) > 0 + - name: Determine new master version hosts: oo_first_master tasks: -- cgit v1.2.3 From 867a372321270311946375d71f7d08aab2b56d67 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 3 Nov 2015 14:57:06 -0400 Subject: Add debug output for location of etcd backup. --- playbooks/adhoc/upgrades/upgrade.yml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index e55f1536f..539710469 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -39,6 +39,9 @@ yum: pkg=etcd state=latest - name: Generate etcd backup command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} + - name: Display location of etcd backup + debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" + - name: Upgrade base package on masters hosts: masters -- cgit v1.2.3 From 4a4ecb7e4517dddf989441f0e7d99617917f942d Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 08:14:46 -0400 Subject: Skip fail if enterprise deployment type depending on version. --- playbooks/adhoc/upgrades/upgrade.yml | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 539710469..ffcd2757b 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -98,6 +98,14 @@ fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) +- name: Verify upgrade can proceed + hosts: masters + tasks: + # Checking the global deployment type rather than host facts, this is about + # what the user is requesting. + - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed" + when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>=')) + - name: Update cluster policy hosts: oo_first_master tasks: -- cgit v1.2.3 From 8ca4308854c44649d2dcfd94f441790c2e5c5d2b Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 08:27:21 -0400 Subject: Use the base package upgrade version so we can check things earlier. --- playbooks/adhoc/upgrades/upgrade.yml | 64 +++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 23 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index ffcd2757b..a959aa881 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -42,7 +42,6 @@ - name: Display location of etcd backup debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" - - name: Upgrade base package on masters hosts: masters roles: @@ -53,6 +52,47 @@ - name: Upgrade base package yum: pkg={{ openshift.common.service_type }}{{ openshift_version }} state=latest +- name: Evaluate oo_first_master + hosts: localhost + vars: + g_masters_group: "{{ 'masters' }}" + tasks: + - name: display all variables set for the current host + debug: + var: hostvars[inventory_hostname] + - name: Evaluate oo_first_master + add_host: + name: "{{ groups[g_masters_group][0] }}" + groups: oo_first_master + ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" + ansible_sudo: "{{ g_sudo | default(omit) }}" + when: g_masters_group in groups and (groups[g_masters_group] | length) > 0 + +# TODO: ideally we would check the new version, without installing it. (some +# kind of yum repoquery? would need to handle openshift -> atomic-openshift +# package rename) +- name: Perform upgrade version checking + hosts: oo_first_master + tasks: + - name: Determine new version + command: > + rpm -q --queryformat '%{version}' {{ openshift.common.service_type }} + register: _new_version + +- name: Ensure AOS 3.0.2 or Origin 1.0.6 + hosts: oo_first_master + tasks: + fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later + when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) + +- name: Verify upgrade can proceed + hosts: oo_first_master + tasks: + # Checking the global deployment type rather than host facts, this is about + # what the user is requesting. + - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed" + when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>=')) + - name: Re-Run cluster configuration to apply latest configuration changes include: ../../common/openshift-cluster/config.yml vars: @@ -84,28 +124,6 @@ - name: Restart node services service: name="{{ openshift.common.service_type }}-node" state=restarted -- name: Determine new master version - hosts: oo_first_master - tasks: - - name: Determine new version - command: > - rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}-master - register: _new_version - -- name: Ensure AOS 3.0.2 or Origin 1.0.6 - hosts: oo_first_master - tasks: - fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later - when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) - -- name: Verify upgrade can proceed - hosts: masters - tasks: - # Checking the global deployment type rather than host facts, this is about - # what the user is requesting. - - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed" - when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>=')) - - name: Update cluster policy hosts: oo_first_master tasks: -- cgit v1.2.3 From 215a7aacc2fc3df19a64a2a57910516533665423 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 10:41:39 -0400 Subject: Fix bug with not upgrading openshift-master to atomic-openshift-master. Removing the full call to config resulted in rpms not getting upgraded. Config was doing a yum update of everything, which picks up the atomic-openshift-master obsoleting openshift-master. The actual yum call changed here would not. Instead we switch to a direct call to yum which correctly picks up the obsoletes and updates to atomic-openshift packages. --- playbooks/adhoc/upgrades/upgrade.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 8c1138797..0f505bf7d 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -57,9 +57,6 @@ vars: g_masters_group: "{{ 'masters' }}" tasks: - - name: display all variables set for the current host - debug: - var: hostvars[inventory_hostname] - name: Evaluate oo_first_master add_host: name: "{{ groups[g_masters_group][0] }}" @@ -93,15 +90,6 @@ - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed" when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>=')) - #- name: Re-Run cluster configuration to apply latest configuration changes - # include: ../../common/openshift-cluster/config.yml - # vars: - # g_etcd_group: "{{ 'etcd' }}" - # g_masters_group: "{{ 'masters' }}" - # g_nodes_group: "{{ 'nodes' }}" - # openshift_cluster_id: "{{ cluster_id | default('default') }}" - # openshift_deployment_type: "{{ deployment_type }}" - - name: Upgrade masters hosts: masters vars: @@ -109,8 +97,11 @@ tasks: - name: Upgrade to latest available kernel yum: pkg=kernel state=latest + - name: display just the deployment_type variable for the current host + debug: + var: hostvars[inventory_hostname] - name: Upgrade master packages - yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest + command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} - name: Upgrade master configuration. openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master - name: Restart master services @@ -124,7 +115,7 @@ - openshift_facts tasks: - name: Upgrade node packages - yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest + command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} - name: Restart node services service: name="{{ openshift.common.service_type }}-node" state=restarted -- cgit v1.2.3 From abab19316ff270d090e27a1fb576b78889e5563c Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 15:49:57 -0400 Subject: Disable proxy cert config upgrade until certs being generated. --- playbooks/adhoc/upgrades/library/openshift_upgrade_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 95c7b1664..357f244e0 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -49,11 +49,12 @@ def upgrade_master_3_0_to_3_1(backup): changed = True # Add the new master proxy client certs: - if 'proxyClientInfo' not in config['kubernetesMasterConfig']: - config['kubernetesMasterConfig']['proxyClientInfo'] = { - 'certFile': 'master.proxy-client.crt', - 'keyFile': 'master.proxy-client.key' - } + # TODO: re-enable this once these certs are generated during upgrade: +# if 'proxyClientInfo' not in config['kubernetesMasterConfig']: +# config['kubernetesMasterConfig']['proxyClientInfo'] = { +# 'certFile': 'master.proxy-client.crt', +# 'keyFile': 'master.proxy-client.key' +# } if changed: if backup: -- cgit v1.2.3 From 922cc828e959eb9bc0438890a876e7830cd73717 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 16:07:58 -0400 Subject: Pylint fixes for config upgrade module. --- .../upgrades/library/openshift_upgrade_config.py | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 357f244e0..60f4fd8b8 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -2,14 +2,14 @@ # -*- coding: utf-8 -*- # vim: expandtab:tabstop=4:shiftwidth=4 +"""Ansible module for modifying OpenShift configs during an upgrade""" + import os import shutil import yaml from datetime import datetime -"""Ansible module for modifying OpenShift configs during an upgrade""" - DOCUMENTATION = ''' --- module: openshift_upgrade_config @@ -21,6 +21,7 @@ EXAMPLES = ''' ''' def get_cfg_dir(): + """Return the correct config directory to use.""" cfg_path = '/etc/origin/' if not os.path.exists(cfg_path): cfg_path = '/etc/openshift/' @@ -28,25 +29,26 @@ def get_cfg_dir(): def upgrade_master_3_0_to_3_1(backup): + """Main upgrade method for 3.0 to 3.1.""" changed = False # Facts do not get transferred to the hosts where custom modules run, # need to make some assumptions here. master_config = os.path.join(get_cfg_dir(), 'master/master-config.yaml') - f = open(master_config, 'r') - config = yaml.safe_load(f.read()) - f.close() + master_cfg_file = open(master_config, 'r') + config = yaml.safe_load(master_cfg_file.read()) + master_cfg_file.close() # Remove v1beta3 from apiLevels: if 'apiLevels' in config and \ 'v1beta3' in config['apiLevels']: - config['apiLevels'].remove('v1beta3') - changed = True + config['apiLevels'].remove('v1beta3') + changed = True if 'apiLevels' in config['kubernetesMasterConfig'] and \ 'v1beta3' in config['kubernetesMasterConfig']['apiLevels']: - config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') - changed = True + config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') + changed = True # Add the new master proxy client certs: # TODO: re-enable this once these certs are generated during upgrade: @@ -61,7 +63,7 @@ def upgrade_master_3_0_to_3_1(backup): timestamp = datetime.now().strftime('%Y%m%d%H%M%S') basedir = os.path.split(master_config)[0] backup_file = os.path.join(basedir, 'master-config.yaml.bak-%s' - % timestamp) + % timestamp) shutil.copyfile(master_config, backup_file) # Write the modified config: out_file = open(master_config, 'w') @@ -72,6 +74,7 @@ def upgrade_master_3_0_to_3_1(backup): def upgrade_master(from_version, to_version, backup): + """Upgrade entry point.""" if from_version == '3.0': if to_version == '3.1': return upgrade_master_3_0_to_3_1(backup) -- cgit v1.2.3 From 75d357c9bba120221a5f996660aa4fdbb87f1fa8 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 4 Nov 2015 16:10:07 -0400 Subject: Override hosts deployment_type fact for version we're upgrading to. --- playbooks/adhoc/upgrades/upgrade.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 0f505bf7d..09f991b1d 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,4 +1,14 @@ --- +- name: Update deployment type + hosts: OSEv3 + roles: + - openshift_facts + post_tasks: # technically tasks are run after roles, but post_tasks is a bit more explicit. + - openshift_facts: + role: common + local_facts: + deployment_type: "{{ deployment_type }}" + - name: Verify upgrade can proceed hosts: masters tasks: -- cgit v1.2.3 From 482c47e87768d7ab4a157c8768283459b6b4bddd Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Wed, 4 Nov 2015 19:46:41 -0500 Subject: Remove pacemaker bits. --- playbooks/adhoc/uninstall.yml | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml index 0503b7cd4..5b10f856c 100644 --- a/playbooks/adhoc/uninstall.yml +++ b/playbooks/adhoc/uninstall.yml @@ -45,6 +45,7 @@ - origin-master-api - origin-master-controllers - origin-node + - pcsd - yum: name={{ item }} state=absent when: not is_atomic | bool @@ -59,6 +60,7 @@ - atomic-openshift-node - atomic-openshift-sdn-ovs - etcd + - corosync - openshift - openshift-master - openshift-node @@ -69,6 +71,8 @@ - origin-master - origin-node - origin-sdn-ovs + - pacemaker + - pcs - tuned-profiles-atomic-enterprise-node - tuned-profiles-atomic-openshift-node - tuned-profiles-openshift-node @@ -139,6 +143,7 @@ - /etc/ansible/facts.d/openshift.fact - /etc/atomic-enterprise - /etc/etcd + - /etc/corosync - /etc/openshift - /etc/openshift-sdn - /etc/origin @@ -157,3 +162,4 @@ - /var/lib/etcd - /var/lib/openshift - /var/lib/origin + - /var/lib/pacemaker -- cgit v1.2.3 From af928b32bf978e5257fa4bb326559bea3e6bee08 Mon Sep 17 00:00:00 2001 From: Brenton Leanhardt Date: Thu, 5 Nov 2015 11:57:45 -0500 Subject: The uninstall playbook needs to remove /run/openshift-sdn If this isn't removed lbr0 won't be recreated by the Node after the following install. --- playbooks/adhoc/uninstall.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml index 5b10f856c..e05ab43f8 100644 --- a/playbooks/adhoc/uninstall.yml +++ b/playbooks/adhoc/uninstall.yml @@ -140,10 +140,11 @@ - file: path={{ item }} state=absent with_items: + - "~{{ ansible_ssh_user }}/.kube" - /etc/ansible/facts.d/openshift.fact - /etc/atomic-enterprise - - /etc/etcd - /etc/corosync + - /etc/etcd - /etc/openshift - /etc/openshift-sdn - /etc/origin @@ -156,10 +157,13 @@ - /etc/sysconfig/origin-master - /etc/sysconfig/origin-node - /root/.kube - - "~{{ ansible_ssh_user }}/.kube" + - /run/openshift-sdn - /usr/share/openshift/examples - /var/lib/atomic-enterprise - /var/lib/etcd - /var/lib/openshift - /var/lib/origin - /var/lib/pacemaker + + - name: restart docker + service: name=docker state=restarted -- cgit v1.2.3 From fe4e9a4ca7028aa877fdd3895225a67b026aea11 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 5 Nov 2015 08:35:53 -0400 Subject: Upgrade improvements - Push config dir logic out of module and use host variables instead. - Backup master config with ansible utility. - Add error handling for the upgrade config module. - Add verbose option to installer. - Return details on what we changed when upgrading config. - Cleanup use of first master. - Don't install upgrade rpms to check what version we'll upgrade to. --- .../upgrades/library/openshift_upgrade_config.py | 53 +++++++-------- playbooks/adhoc/upgrades/upgrade.yml | 78 +++++++++------------- 2 files changed, 56 insertions(+), 75 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 60f4fd8b8..0894efa52 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -5,11 +5,8 @@ """Ansible module for modifying OpenShift configs during an upgrade""" import os -import shutil import yaml -from datetime import datetime - DOCUMENTATION = ''' --- module: openshift_upgrade_config @@ -20,21 +17,14 @@ requirements: [ ] EXAMPLES = ''' ''' -def get_cfg_dir(): - """Return the correct config directory to use.""" - cfg_path = '/etc/origin/' - if not os.path.exists(cfg_path): - cfg_path = '/etc/openshift/' - return cfg_path - -def upgrade_master_3_0_to_3_1(backup): +def upgrade_master_3_0_to_3_1(module, config_base, backup): """Main upgrade method for 3.0 to 3.1.""" - changed = False + changes = [] # Facts do not get transferred to the hosts where custom modules run, # need to make some assumptions here. - master_config = os.path.join(get_cfg_dir(), 'master/master-config.yaml') + master_config = os.path.join(config_base, 'master/master-config.yaml') master_cfg_file = open(master_config, 'r') config = yaml.safe_load(master_cfg_file.read()) @@ -45,6 +35,7 @@ def upgrade_master_3_0_to_3_1(backup): 'v1beta3' in config['apiLevels']: config['apiLevels'].remove('v1beta3') changed = True + changes.append("master-config.yaml: removed v1beta3 from apiLevels") if 'apiLevels' in config['kubernetesMasterConfig'] and \ 'v1beta3' in config['kubernetesMasterConfig']['apiLevels']: config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') @@ -57,27 +48,26 @@ def upgrade_master_3_0_to_3_1(backup): # 'certFile': 'master.proxy-client.crt', # 'keyFile': 'master.proxy-client.key' # } +# changes.append("master-config.yaml: added proxyClientInfo") - if changed: + if len(changes) > 0: if backup: - timestamp = datetime.now().strftime('%Y%m%d%H%M%S') - basedir = os.path.split(master_config)[0] - backup_file = os.path.join(basedir, 'master-config.yaml.bak-%s' - % timestamp) - shutil.copyfile(master_config, backup_file) + # TODO: Check success: + module.backup_local(master_config) + # Write the modified config: out_file = open(master_config, 'w') out_file.write(yaml.safe_dump(config, default_flow_style=False)) out_file.close() - return changed + return changes -def upgrade_master(from_version, to_version, backup): +def upgrade_master(module, config_base, from_version, to_version, backup): """Upgrade entry point.""" if from_version == '3.0': if to_version == '3.1': - return upgrade_master_3_0_to_3_1(backup) + return upgrade_master_3_0_to_3_1(module, config_base, backup) def main(): @@ -89,6 +79,7 @@ def main(): module = AnsibleModule( argument_spec=dict( + config_base=dict(required=True), from_version=dict(required=True, choices=['3.0']), to_version=dict(required=True, choices=['3.1']), role=dict(required=True, choices=['master']), @@ -101,12 +92,18 @@ def main(): to_version = module.params['to_version'] role = module.params['role'] backup = module.params['backup'] - - changed = False - if role == 'master': - changed = upgrade_master(from_version, to_version, backup) - - return module.exit_json(changed=changed) + config_base = module.params['config_base'] + + try: + changes = [] + if role == 'master': + changes = upgrade_master(module, config_base, from_version, + to_version, backup) + + changed = len(changes) > 0 + return module.exit_json(changed=changed, changes=changes) + except Exception, e: + return module.fail_json(msg=str(e)) # ignore pylint errors related to the module_utils import # pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 09f991b1d..c113c7ab2 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,4 +1,12 @@ --- +- name: Verify upgrade can proceed + hosts: masters + tasks: + # Checking the global deployment type rather than host facts, this is about + # what the user is requesting. + - fail: msg="Deployment type enterprise not supported for upgrade" + when: deployment_type == "enterprise" + - name: Update deployment type hosts: OSEv3 roles: @@ -9,14 +17,6 @@ local_facts: deployment_type: "{{ deployment_type }}" -- name: Verify upgrade can proceed - hosts: masters - tasks: - # Checking the global deployment type rather than host facts, this is about - # what the user is requesting. - - fail: msg="Deployment type enterprise not supported for upgrade" - when: deployment_type == "enterprise" - - name: Backup etcd hosts: masters vars: @@ -52,48 +52,35 @@ - name: Display location of etcd backup debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" -- name: Upgrade base package on masters - hosts: masters - roles: - - openshift_facts - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - tasks: - - name: Upgrade base package - yum: pkg={{ openshift.common.service_type }}{{ openshift_version }} state=latest - -- name: Evaluate oo_first_master - hosts: localhost - vars: - g_masters_group: "{{ 'masters' }}" - tasks: - - name: Evaluate oo_first_master - add_host: - name: "{{ groups[g_masters_group][0] }}" - groups: oo_first_master - ansible_ssh_user: "{{ g_ssh_user | default(omit) }}" - ansible_sudo: "{{ g_sudo | default(omit) }}" - when: g_masters_group in groups and (groups[g_masters_group] | length) > 0 - -# TODO: ideally we would check the new version, without installing it. (some -# kind of yum repoquery? would need to handle openshift -> atomic-openshift -# package rename) - name: Perform upgrade version checking - hosts: oo_first_master + hosts: masters[0] tasks: - - name: Determine new version + - name: Determine available version + shell: > + yum list available {{ openshift.common.service_type }} | tail -n 1 | cut -f 2 -d " " | cut -f 1 -d "-" + register: _new_version + - debug: var=_new_version + # The above check will return nothing if the package is already installed, + # and we may be re-running upgrade due to a failure. + - name: Determine installed version command: > rpm -q --queryformat '%{version}' {{ openshift.common.service_type }} register: _new_version + when: _new_version.stdout == "" + # Fail if we still don't know: + - debug: var=_new_version + - name: Verify upgrade version + fail: Unable to determine upgrade version for {{ openshift.common.service_type }} + when: _new_version.stdout == "" - name: Ensure AOS 3.0.2 or Origin 1.0.6 - hosts: oo_first_master + hosts: masters[0] tasks: fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) - name: Verify upgrade can proceed - hosts: oo_first_master + hosts: masters[0] tasks: # Checking the global deployment type rather than host facts, this is about # what the user is requesting. @@ -107,13 +94,10 @@ tasks: - name: Upgrade to latest available kernel yum: pkg=kernel state=latest - - name: display just the deployment_type variable for the current host - debug: - var: hostvars[inventory_hostname] - name: Upgrade master packages command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} - name: Upgrade master configuration. - openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master + openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master config_base={{ hostvars[inventory_hostname].openshift.common.config_base }} - name: Restart master services service: name="{{ openshift.common.service_type}}-master" state=restarted @@ -130,7 +114,7 @@ service: name="{{ openshift.common.service_type }}-node" state=restarted - name: Update cluster policy - hosts: oo_first_master + hosts: masters[0] tasks: - name: oadm policy reconcile-cluster-roles --confirm command: > @@ -138,7 +122,7 @@ policy reconcile-cluster-roles --confirm - name: Update cluster policy bindings - hosts: oo_first_master + hosts: masters[0] tasks: - name: oadm policy reconcile-cluster-role-bindings --confirm command: > @@ -151,7 +135,7 @@ when: ( _new_version.stdout | version_compare('1.0.6', '>') and _new_version.stdout | version_compare('3.0','<') ) or _new_version.stdout | version_compare('3.0.2','>') - name: Upgrade default router - hosts: oo_first_master + hosts: masters[0] vars: - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" @@ -189,7 +173,7 @@ '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' - name: Upgrade default - hosts: oo_first_master + hosts: masters[0] vars: - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" @@ -207,7 +191,7 @@ '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' - name: Update image streams and templates - hosts: oo_first_master + hosts: masters[0] vars: openshift_examples_import_command: "update" openshift_deployment_type: "{{ deployment_type }}" -- cgit v1.2.3 From b65403b8ac3cd0eea46179d4758f6f0be5929728 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Wed, 4 Nov 2015 22:55:25 -0500 Subject: Further upgrade improvements - Restart masters post reconcile - generate missing master certs and sync to masters --- playbooks/adhoc/upgrades/upgrade.yml | 214 +++++++++++++++++++++++++++-------- 1 file changed, 168 insertions(+), 46 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index c113c7ab2..7433dc9c0 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,22 +1,17 @@ --- - name: Verify upgrade can proceed - hosts: masters + hosts: masters[0] + gather_facts: no tasks: - # Checking the global deployment type rather than host facts, this is about - # what the user is requesting. + # Checking the global deployment type rather than host facts, this is about + # what the user is requesting. - fail: msg="Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" -- name: Update deployment type - hosts: OSEv3 - roles: - - openshift_facts - post_tasks: # technically tasks are run after roles, but post_tasks is a bit more explicit. - - openshift_facts: - role: common - local_facts: - deployment_type: "{{ deployment_type }}" - +# TODO: etcd is only guaranteed to be run on the master if embedded etcd is +# used, we should have a task to create a new group consisting of the single +# master running embedded etcd or an external etcd cluster specified by the +# etcd group in the inventory (which could also be the masters) - name: Backup etcd hosts: masters vars: @@ -27,9 +22,11 @@ tasks: - stat: path=/var/lib/openshift register: var_lib_openshift + - name: Create origin symlink if necessary file: src=/var/lib/openshift/ dest=/var/lib/origin state=link when: var_lib_openshift.stat.exists == True + - name: Check available disk space for etcd backup # We assume to be using the data dir for all backups. shell: > @@ -43,14 +40,51 @@ when: embedded_etcd | bool - name: Abort if insufficient disk space for etcd backup - fail: msg="{{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, {{ avail_disk.stdout }} Kb available." + fail: + msg: > + {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, + {{ avail_disk.stdout }} Kb available. when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) + - name: Install etcd (for etcdctl) - yum: pkg=etcd state=latest + yum: + pkg: etcd + state: latest + - name: Generate etcd backup - command: etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} + command: > + etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} + --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} + - name: Display location of etcd backup - debug: msg="Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" + debug: + msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" + +- name: Update deployment type + hosts: OSEv3 + roles: + - openshift_facts + post_tasks: + - openshift_facts: + role: common + local_facts: + deployment_type: "{{ deployment_type }}" + +- name: Upgrade base package on masters + hosts: masters + roles: + - openshift_facts + vars: + openshift_version: "{{ openshift_pkg_version | default('') }}" + tasks: + - name: Upgrade base package + yum: + pkg: "{{ openshift.common.service_type }}{{ openshift_version }}" + state: latest + +# TODO: ideally we would check the new version, without installing it. (some +# kind of yum repoquery? would need to handle openshift -> atomic-openshift +# package rename) - name: Perform upgrade version checking hosts: masters[0] @@ -73,19 +107,13 @@ fail: Unable to determine upgrade version for {{ openshift.common.service_type }} when: _new_version.stdout == "" -- name: Ensure AOS 3.0.2 or Origin 1.0.6 - hosts: masters[0] - tasks: - fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later - when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') ) + - fail: + msg: This playbook requires Atomic OpenShift 3.0.2 or later + when: deployment_type in ['openshift_enterprise', 'atomic-enterprise'] and g_new_version.stdout | version_compare('3.0','>=') and g_new_version.stdout | version_compare('3.0.2','<') -- name: Verify upgrade can proceed - hosts: masters[0] - tasks: - # Checking the global deployment type rather than host facts, this is about - # what the user is requesting. - - fail: msg="Deployment type 'enterprise' must be updated to 'openshift-enterprise' for upgrade to proceed" - when: deployment_type == "enterprise" and (_new_version.stdout | version_compare('1.0.7', '>=') or _new_version.stdout | version_compare('3.1', '>=')) + - fail: + msg: This playbook requires Origin 1.0.6 or later + when: deployment_type == 'origin' - name: Upgrade masters hosts: masters @@ -93,13 +121,104 @@ openshift_version: "{{ openshift_pkg_version | default('') }}" tasks: - name: Upgrade to latest available kernel - yum: pkg=kernel state=latest + yum: + pkg: kernel + state: latest + + - name: display just the deployment_type variable for the current host + debug: + var: hostvars[inventory_hostname].openshift.common.deployment_type + - name: Upgrade master packages command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} - - name: Upgrade master configuration. - openshift_upgrade_config: from_version=3.0 to_version=3.1 role=master config_base={{ hostvars[inventory_hostname].openshift.common.config_base }} - - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted + + - name: Ensure python-yaml present for config upgrade + yum: + pkg: python-yaml + state: installed + + - name: Upgrade master configuration + openshift_upgrade_config: + from_version: '3.0' + to_version: '3.1' + role: master + + - set_fact: + master_certs_missing: True + master_cert_subdir: master-{{ openshift.common.hostname }} + master_cert_config_dir: "{{ openshift.common.config_base }}/master" + +- name: Create temp directory for syncing certs + hosts: localhost + gather_facts: no + tasks: + - name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX + register: g_master_mktemp + changed_when: False + +- name: Generate missing master certificates + hosts: masters[0] + vars: + master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" + masters_needing_certs: "{{ hostvars + | oo_select_keys(groups.masters) + | difference([groups.masters.0]) }}" + sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_deployment_type: "{{ deployment_type }}" + roles: + - openshift_master_certificates + post_tasks: + - name: Remove generated etcd client certs when using external etcd + file: + path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: absent + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + with_nested: + - masters_needing_certs + - - master.etcd-client.crt + - master.etcd-client.key + + - name: Create a tarball of the master certs + command: > + tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz + -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . + args: + creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" + with_items: masters_needing_certs + + - name: Retrieve the master cert tarball from the master + fetch: + src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" + dest: "{{ sync_tmpdir }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + with_items: masters_needing_certs + + +- name: Sync certs and restart masters post configuration change + hosts: masters + vars: + sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + tasks: + - name: Unarchive the tarball on the master + unarchive: + src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz" + dest: "{{ master_cert_config_dir }}" + when: inventory_hostname != groups.masters.0 + + - name: Restart master services + service: name="{{ openshift.common.service_type}}-master" state=restarted + + +- name: Delete temporary directory on localhost + hosts: localhost + gather_facts: no + tasks: + - file: name={{ g_master_mktemp.stdout }} state=absent + changed_when: False + - name: Upgrade nodes hosts: nodes @@ -113,17 +232,17 @@ - name: Restart node services service: name="{{ openshift.common.service_type }}-node" state=restarted -- name: Update cluster policy +- name: Update cluster policy and policy bindings hosts: masters[0] + vars: + origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version.stdout | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version.stdout | version_compare('3.0.2','>') }}" tasks: - name: oadm policy reconcile-cluster-roles --confirm command: > {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig policy reconcile-cluster-roles --confirm -- name: Update cluster policy bindings - hosts: masters[0] - tasks: - name: oadm policy reconcile-cluster-role-bindings --confirm command: > {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig @@ -132,11 +251,20 @@ --exclude-groups=system:unauthenticated --exclude-users=system:anonymous --additive-only=true --confirm - when: ( _new_version.stdout | version_compare('1.0.6', '>') and _new_version.stdout | version_compare('3.0','<') ) or _new_version.stdout | version_compare('3.0.2','>') + when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool + + +- name: Restart masters post reconcile + hosts: masters + tasks: + - name: Restart master services + service: name="{{ openshift.common.service_type}}-master" state=restarted -- name: Upgrade default router + +- name: Upgrade default router and registry hosts: masters[0] vars: + - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" tasks: @@ -172,12 +300,6 @@ {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' -- name: Upgrade default - hosts: masters[0] - vars: - - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" - tasks: - name: Check for default registry command: > {{ oc_cmd }} get -n default dc/docker-registry -- cgit v1.2.3 From eb4cfd5702e675219ae0d36df667e0029b353921 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 5 Nov 2015 14:22:13 -0400 Subject: Handle backups for separate etcd hosts if necessary. --- playbooks/adhoc/upgrades/upgrade.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 7433dc9c0..d9abff040 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -1,4 +1,9 @@ --- +- name: Load master facts + hosts: masters + roles: + - openshift_facts + - name: Verify upgrade can proceed hosts: masters[0] gather_facts: no @@ -8,18 +13,30 @@ - fail: msg="Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" -# TODO: etcd is only guaranteed to be run on the master if embedded etcd is -# used, we should have a task to create a new group consisting of the single -# master running embedded etcd or an external etcd cluster specified by the -# etcd group in the inventory (which could also be the masters) +- name: Evaluate etcd_hosts + hosts: localhost + tasks: + - name: Evaluate etcd hosts + add_host: + name: "{{ groups.masters.0 }}" + groups: etcd_hosts + when: hostvars[groups.masters.0].openshift.master.embedded_etcd | bool + - name: Evaluate etcd hosts + add_host: + name: "{{ item }}" + groups: etcd_hosts + with_items: groups.etcd + when: not hostvars[groups.masters.0].openshift.master.embedded_etcd | bool + - name: Backup etcd - hosts: masters + hosts: etcd_hosts vars: embedded_etcd: "{{ openshift.master.embedded_etcd }}" timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" roles: - openshift_facts tasks: + - stat: path=/var/lib/openshift register: var_lib_openshift @@ -142,6 +159,7 @@ from_version: '3.0' to_version: '3.1' role: master + config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - set_fact: master_certs_missing: True @@ -235,8 +253,8 @@ - name: Update cluster policy and policy bindings hosts: masters[0] vars: - origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version.stdout | version_compare('1.0.6', '>') }}" - ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version.stdout | version_compare('3.0.2','>') }}" + origin_reconcile_bindings: "{{ deployment_type == 'origin' and gg_new_version.stdout | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and gg_new_version.stdout | version_compare('3.0.2','>') }}" tasks: - name: oadm policy reconcile-cluster-roles --confirm command: > @@ -264,8 +282,8 @@ - name: Upgrade default router and registry hosts: masters[0] vars: - - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" - - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}" + - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version.stdout ) }}" + - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version.stdout ) }}" - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" tasks: - name: Check for default router -- cgit v1.2.3 From e9680cc1020f9c54221993b3ae816b046d92bafc Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Thu, 5 Nov 2015 15:42:30 -0500 Subject: Additional upgrade enhancements - rework the version checking - provide better safety if the apiLevel attributes are missing - ensure a list of api levels are present - remove a list of api levels - pylint fixes --- playbooks/adhoc/upgrades/files/versions.sh | 8 ++ .../upgrades/library/openshift_upgrade_config.py | 92 ++++++++++++++++------ playbooks/adhoc/upgrades/upgrade.yml | 73 ++++++----------- 3 files changed, 101 insertions(+), 72 deletions(-) create mode 100644 playbooks/adhoc/upgrades/files/versions.sh (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/files/versions.sh b/playbooks/adhoc/upgrades/files/versions.sh new file mode 100644 index 000000000..01ea1d91a --- /dev/null +++ b/playbooks/adhoc/upgrades/files/versions.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +yum_installed=$(yum list installed "$@" | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | awk '{ print $2 }' | tr '\n' ' ') +yum_available=$(yum list available "$@" | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | awk '{ print $2 }' | tr '\n' ' ') + +echo "---" +echo "curr_version: ${yum_installed}" +echo "avail_version: ${yum_available}" diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 0894efa52..33eb40d7d 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -17,8 +17,40 @@ requirements: [ ] EXAMPLES = ''' ''' +def modify_api_levels(level_list, remove, ensure, msg_prepend='', + msg_append=''): + """ modify_api_levels """ + changed = False + changes = [] + + if not isinstance(remove, list): + remove = [] + + if not isinstance(ensure, list): + ensure = [] + + if not isinstance(level_list, list): + new_list = [] + changed = True + changes.append("%s created missing %s" % (msg_prepend, msg_append)) + else: + new_list = level_list + for level in remove: + if level in new_list: + new_list.remove(level) + changed = True + changes.append("%s removed %s %s" % (msg_prepend, level, msg_append)) + + for level in ensure: + if level not in new_list: + new_list.append(level) + changed = True + changes.append("%s added %s %s" % (msg_prepend, level, msg_append)) -def upgrade_master_3_0_to_3_1(module, config_base, backup): + return {'new_list': new_list, 'changed': changed, 'changes': changes} + + +def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): """Main upgrade method for 3.0 to 3.1.""" changes = [] @@ -30,30 +62,38 @@ def upgrade_master_3_0_to_3_1(module, config_base, backup): config = yaml.safe_load(master_cfg_file.read()) master_cfg_file.close() - # Remove v1beta3 from apiLevels: - if 'apiLevels' in config and \ - 'v1beta3' in config['apiLevels']: - config['apiLevels'].remove('v1beta3') - changed = True - changes.append("master-config.yaml: removed v1beta3 from apiLevels") - if 'apiLevels' in config['kubernetesMasterConfig'] and \ - 'v1beta3' in config['kubernetesMasterConfig']['apiLevels']: - config['kubernetesMasterConfig']['apiLevels'].remove('v1beta3') - changed = True - # Add the new master proxy client certs: - # TODO: re-enable this once these certs are generated during upgrade: -# if 'proxyClientInfo' not in config['kubernetesMasterConfig']: -# config['kubernetesMasterConfig']['proxyClientInfo'] = { -# 'certFile': 'master.proxy-client.crt', -# 'keyFile': 'master.proxy-client.key' -# } -# changes.append("master-config.yaml: added proxyClientInfo") + # Remove unsupported api versions and ensure supported api versions from + # master config + unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3'] + supported_levels = ['v1'] + + result = modify_api_levels(config.get('apiLevels'), unsupported_levels, + supported_levels, 'master-config.yaml:', 'from apiLevels') + if result['changed']: + config['apiLevels'] = result['new_list'] + changes.append(result['changes']) + + if 'kubernetesMasterConfig' in config: + result = modify_api_levels(config['kubernetesMasterConfig'].get('apiLevels'), + unsupported_levels, supported_levels, 'master-config.yaml:', + 'from apiLevels') + if result['changed']: + config['kubernetesMasterConfig']['apiLevels'] = result['new_list'] + changes.append(result['changes']) + + # Add proxyClientInfo to master-config + if 'proxyClientInfo' not in config['kubernetesMasterConfig']: + config['kubernetesMasterConfig']['proxyClientInfo'] = { + 'certFile': 'master.proxy-client.crt', + 'keyFile': 'master.proxy-client.key' + } + changes.append("master-config.yaml: added proxyClientInfo") if len(changes) > 0: if backup: # TODO: Check success: - module.backup_local(master_config) + ansible_module.backup_local(master_config) # Write the modified config: out_file = open(master_config, 'w') @@ -63,18 +103,19 @@ def upgrade_master_3_0_to_3_1(module, config_base, backup): return changes -def upgrade_master(module, config_base, from_version, to_version, backup): +def upgrade_master(ansible_module, config_base, from_version, to_version, backup): """Upgrade entry point.""" if from_version == '3.0': if to_version == '3.1': - return upgrade_master_3_0_to_3_1(module, config_base, backup) + return upgrade_master_3_0_to_3_1(ansible_module, config_base, backup) def main(): """ main """ # disabling pylint errors for global-variable-undefined and invalid-name # for 'global module' usage, since it is required to use ansible_facts - # pylint: disable=global-variable-undefined, invalid-name + # pylint: disable=global-variable-undefined, invalid-name, + # redefined-outer-name global module module = AnsibleModule( @@ -98,10 +139,13 @@ def main(): changes = [] if role == 'master': changes = upgrade_master(module, config_base, from_version, - to_version, backup) + to_version, backup) changed = len(changes) > 0 return module.exit_json(changed=changed, changes=changes) + + # ignore broad-except error to avoid stack trace to ansible user + # pylint: disable=broad-except except Exception, e: return module.fail_json(msg=str(e)) diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index d9abff040..3b3609dca 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -87,50 +87,29 @@ local_facts: deployment_type: "{{ deployment_type }}" -- name: Upgrade base package on masters - hosts: masters - roles: - - openshift_facts - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - tasks: - - name: Upgrade base package - yum: - pkg: "{{ openshift.common.service_type }}{{ openshift_version }}" - state: latest - -# TODO: ideally we would check the new version, without installing it. (some -# kind of yum repoquery? would need to handle openshift -> atomic-openshift -# package rename) - name: Perform upgrade version checking hosts: masters[0] tasks: - - name: Determine available version - shell: > - yum list available {{ openshift.common.service_type }} | tail -n 1 | cut -f 2 -d " " | cut -f 1 -d "-" - register: _new_version - - debug: var=_new_version - # The above check will return nothing if the package is already installed, - # and we may be re-running upgrade due to a failure. - - name: Determine installed version - command: > - rpm -q --queryformat '%{version}' {{ openshift.common.service_type }} - register: _new_version - when: _new_version.stdout == "" - # Fail if we still don't know: - - debug: var=_new_version - - name: Verify upgrade version - fail: Unable to determine upgrade version for {{ openshift.common.service_type }} - when: _new_version.stdout == "" - - - fail: - msg: This playbook requires Atomic OpenShift 3.0.2 or later - when: deployment_type in ['openshift_enterprise', 'atomic-enterprise'] and g_new_version.stdout | version_compare('3.0','>=') and g_new_version.stdout | version_compare('3.0.2','<') - - - fail: - msg: This playbook requires Origin 1.0.6 or later - when: deployment_type == 'origin' + - name: Clean yum cache + command: yum clean all + + - name: Determine available versions + script: files/versions.sh {{ openshift.common.service_type }} openshift + register: g_versions_result + + - set_fact: + g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}" + + - set_fact: + g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" + + - fail: This playbook requires Origin 1.0.6 or later + when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') + + - fail: This playbook requires Atomic OpenShift 3.0.2 or later + when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2','<') + - name: Upgrade masters hosts: masters @@ -142,10 +121,6 @@ pkg: kernel state: latest - - name: display just the deployment_type variable for the current host - debug: - var: hostvars[inventory_hostname].openshift.common.deployment_type - - name: Upgrade master packages command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} @@ -154,6 +129,8 @@ pkg: python-yaml state: installed + - debug: var=hostvars[inventory_hostname].openshift.common.config_base + - name: Upgrade master configuration openshift_upgrade_config: from_version: '3.0' @@ -253,8 +230,8 @@ - name: Update cluster policy and policy bindings hosts: masters[0] vars: - origin_reconcile_bindings: "{{ deployment_type == 'origin' and gg_new_version.stdout | version_compare('1.0.6', '>') }}" - ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and gg_new_version.stdout | version_compare('3.0.2','>') }}" + origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}" + ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version | version_compare('3.0.2','>') }}" tasks: - name: oadm policy reconcile-cluster-roles --confirm command: > @@ -282,8 +259,8 @@ - name: Upgrade default router and registry hosts: masters[0] vars: - - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version.stdout ) }}" - - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version.stdout ) }}" + - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}" + - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}" - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" tasks: - name: Check for default router -- cgit v1.2.3 From 7b405f03f75b3c03a6016024ee757c1645bf234c Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Thu, 5 Nov 2015 17:34:12 -0500 Subject: add master_hostnames definition for upgrade --- playbooks/adhoc/upgrades/upgrade.yml | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 3b3609dca..e63add4d1 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -155,6 +155,10 @@ - name: Generate missing master certificates hosts: masters[0] vars: + master_hostnames: "{{ hostvars + | oo_select_keys(groups.masters) + | oo_collect('openshift.common.all_hostnames') + | oo_flatten | unique }}" master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" masters_needing_certs: "{{ hostvars | oo_select_keys(groups.masters) -- cgit v1.2.3 From d0fbf1598a9ee6e52a4fae4c5922c580df06a78d Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Thu, 5 Nov 2015 18:09:00 -0500 Subject: Start to handle pacemaker ha during upgrade --- playbooks/adhoc/upgrades/upgrade.yml | 54 ++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index e63add4d1..e807d3fa8 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -6,12 +6,17 @@ - name: Verify upgrade can proceed hosts: masters[0] + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" gather_facts: no tasks: # Checking the global deployment type rather than host facts, this is about # what the user is requesting. - fail: msg="Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" + # Pacemaker is currently the only supported upgrade path for multiple masters + - fail: msg="openshift_master_cluster_method must be set to 'pacemaker'" + when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker")) - name: Evaluate etcd_hosts hosts: localhost @@ -182,8 +187,6 @@ command: > tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . - args: - creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" with_items: masters_needing_certs - name: Retrieve the master cert tarball from the master @@ -195,11 +198,11 @@ validate_checksum: yes with_items: masters_needing_certs - - name: Sync certs and restart masters post configuration change hosts: masters vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" tasks: - name: Unarchive the tarball on the master unarchive: @@ -209,7 +212,41 @@ - name: Restart master services service: name="{{ openshift.common.service_type}}-master" state=restarted + when: not openshift_master_ha | bool +- name: Destroy cluster + hosts: masters[0] + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" + openshift_deployment_type: "{{ deployment_type }}" + pre_tasks: + - name: Check for configured cluster + stat: + path: /etc/corosync/corosync.conf + register: corosync_conf + when: openshift_master_ha | bool + - name: Destroy cluster + command: pcs cluster destroy --all + when: openshift_master_ha | bool and corosync_conf.stat.exists == true + +- name: Start pcsd on masters + hosts: masters + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" + tasks: + - name: Start pcsd + service: name=pcsd state=started + when: openshift_master_ha | bool + +- name: Re-create cluster + hosts: masters[0] + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" + openshift_deployment_type: "{{ deployment_type }}" + omc_cluster_hosts: "{{ groups.masters | join(' ') }}" + roles: + - role: openshift_master_cluster + when: openshift_master_ha | bool - name: Delete temporary directory on localhost hosts: localhost @@ -255,10 +292,21 @@ - name: Restart masters post reconcile hosts: masters + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" tasks: - name: Restart master services service: name="{{ openshift.common.service_type}}-master" state=restarted + when: not openshift_master_ha | bool +- name: Restart cluster post reconcile + hosts: masters[0] + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" + tasks: + - name: Restart master cluster + command: pcs resource restart master + when: openshift_master_ha | bool - name: Upgrade default router and registry hosts: masters[0] -- cgit v1.2.3 From 66791fd954731c7d4286d48683b7fe40288d5000 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 6 Nov 2015 09:49:17 -0400 Subject: Add pre-upgrade script to be run on first master. Script currently just checks for port names that are no longer valid. In theory other checks may be added to this script in the future. Script was originally written by Steve Milner and Andy Goldstein. If the script fails, ansible seems to handle this nicely by default, exiting the upgrade and displaying stderr and stdout, both of which contain useful info on what the problem was. --- playbooks/adhoc/upgrades/files/pre-upgrade-check | 185 +++++++++++++++++++++++ playbooks/adhoc/upgrades/upgrade.yml | 7 + 2 files changed, 192 insertions(+) create mode 100644 playbooks/adhoc/upgrades/files/pre-upgrade-check (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/files/pre-upgrade-check b/playbooks/adhoc/upgrades/files/pre-upgrade-check new file mode 100644 index 000000000..c8ecae399 --- /dev/null +++ b/playbooks/adhoc/upgrades/files/pre-upgrade-check @@ -0,0 +1,185 @@ +#!/usr/bin/env python +""" +Pre-upgrade checks that must be run on a master before proceeding with upgrade. +""" +# This is a script not a python module: +# pylint: disable=invalid-name + +# NOTE: This script should not require any python libs other than what is +# in the standard library. + +__license__ = "ASL 2.0" + +import json +import os +import subprocess +import re + +# The maximum length of container.ports.name +ALLOWED_LENGTH = 15 +# The valid structure of container.ports.name +ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$') +AT_LEAST_ONE_LETTER = re.compile('[a-z]') +# look at OS_PATH for the full path. Default ot 'oc' +OC_PATH = os.getenv('OC_PATH', 'oc') + + +def validate(value): + """ + validate verifies that value matches required conventions + + Rules of container.ports.name validation: + + * must be less that 16 chars + * at least one letter + * only a-z0-9- + * hyphens can not be leading or trailing or next to each other + + :Parameters: + - `value`: Value to validate + """ + if len(value) > ALLOWED_LENGTH: + return False + + if '--' in value: + return False + + # We search since it can be anywhere + if not AT_LEAST_ONE_LETTER.search(value): + return False + + # We match because it must start at the beginning + if not ALLOWED_CHARS.match(value): + return False + return True + + +def list_items(kind): + """ + list_items returns a list of items from the api + + :Parameters: + - `kind`: Kind of item to access + """ + response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind]) + items = json.loads(response) + return items.get("items", []) + + +def get(obj, *paths): + """ + Gets an object + + :Parameters: + - `obj`: A dictionary structure + - `path`: All other non-keyword arguments + """ + ret_obj = obj + for path in paths: + if ret_obj.get(path, None) is None: + return [] + ret_obj = ret_obj[path] + return ret_obj + + +# pylint: disable=too-many-arguments +def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid): + """ + Prints out results in human friendly way. + + :Parameters: + - `namespace`: Namespace of the resource + - `kind`: Kind of the resource + - `item_name`: Name of the resource + - `container_name`: Name of the container. May be "" when kind=Service. + - `port_name`: Name of the port + - `valid`: True if the port is valid + """ + if not valid: + if len(container_name) > 0: + print('%s/%s -n %s (Container="%s" Port="%s")' % ( + kind, item_name, namespace, container_name, port_name)) + else: + print('%s/%s -n %s (Port="%s")' % ( + kind, item_name, namespace, port_name)) + + +def print_validation_header(): + """ + Prints the error header. Should run on the first error to avoid + overwhelming the user. + """ + print """\ +At least one port name does not validate. Valid port names: + + * must be less that 16 chars + * have at least one letter + * only a-z0-9- + * do not start or end with - + * Dashes may not be next to eachother ('--') +""" + + +def main(): + """ + main is the main entry point to this script + """ + try: + # the comma at the end suppresses the newline + print "Checking for oc ...", + subprocess.check_output([OC_PATH, 'whoami']) + print "found" + except: + print( + 'Can not find oc (%s). Override the path with the ' + 'OC_PATH environment variable. Exiting...' % OC_PATH) + raise SystemExit(1) + + # Where the magic happens + first_error = True + for kind, path in [ + ('replicationcontrollers', ("spec", "template", "spec", "containers")), + ('pods', ("spec", "containers")), + ('deploymentconfigs', ("spec", "template", "spec", "containers"))]: + for item in list_items(kind): + namespace = item["metadata"]["namespace"] + item_name = item["metadata"]["name"] + for container in get(item, *path): + container_name = container["name"] + for port in get(container, "ports"): + port_name = port.get("name", None) + if not port_name: + # Unnamed ports are OK + continue + valid = validate(port_name) + if not valid and first_error: + first_error = False + print_validation_header() + pretty_print_errors( + namespace, kind, item_name, + container_name, port_name, valid) + + # Services follow a different flow + for item in list_items('services'): + namespace = item["metadata"]["namespace"] + item_name = item["metadata"]["name"] + for port in get(item, "spec", "ports"): + port_name = port.get("targetPort", None) + if isinstance(port_name, int) or port_name is None: + # Integer only or unnamed ports are OK + continue + valid = validate(port_name) + if not valid and first_error: + first_error = False + print_validation_header() + pretty_print_errors( + namespace, "services", item_name, "", port_name, valid) + + # If we had at least 1 error then exit with 1 + if not first_error: + raise SystemExit(1) + + +if __name__ == '__main__': + main() + diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index e63add4d1..4ca9b94ac 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -13,6 +13,13 @@ - fail: msg="Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" +- name: Run pre-upgrade checks on first master + hosts: masters[0] + tasks: + # If this script errors out ansible will show the default stdout/stderr + # which contains details for the user: + - script: files/pre-upgrade-check + - name: Evaluate etcd_hosts hosts: localhost tasks: -- cgit v1.2.3 From afb03bb7e30cb46bc3fd55fa58888b13171db299 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 6 Nov 2015 12:32:11 -0400 Subject: Fix creation of origin symlink when dir already exists. --- playbooks/adhoc/upgrades/upgrade.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 4ca9b94ac..890a51241 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -47,9 +47,12 @@ - stat: path=/var/lib/openshift register: var_lib_openshift + - stat: path=/var/lib/origin + register: var_lib_origin + - name: Create origin symlink if necessary file: src=/var/lib/openshift/ dest=/var/lib/origin state=link - when: var_lib_openshift.stat.exists == True + when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False - name: Check available disk space for etcd backup # We assume to be using the data dir for all backups. -- cgit v1.2.3 From 941246584a95cc4b5d5619e2b31f75923888f2f6 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 11:36:17 -0500 Subject: Fix apiLevels modifications --- playbooks/adhoc/upgrades/library/openshift_upgrade_config.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index 33eb40d7d..f9ee5b254 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -74,13 +74,9 @@ def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): config['apiLevels'] = result['new_list'] changes.append(result['changes']) - if 'kubernetesMasterConfig' in config: - result = modify_api_levels(config['kubernetesMasterConfig'].get('apiLevels'), - unsupported_levels, supported_levels, 'master-config.yaml:', - 'from apiLevels') - if result['changed']: - config['kubernetesMasterConfig']['apiLevels'] = result['new_list'] - changes.append(result['changes']) + if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']: + config['kubernetesMasterConfig'].remove('apiLevels') + changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels') # Add proxyClientInfo to master-config if 'proxyClientInfo' not in config['kubernetesMasterConfig']: -- cgit v1.2.3 From 050209f3c86a9dccefdf8b9f05898d1503254a2c Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 13:21:54 -0500 Subject: Fix issues related to upgrade packages being unavailable --- playbooks/adhoc/upgrades/files/versions.sh | 6 ++++-- playbooks/adhoc/upgrades/upgrade.yml | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/files/versions.sh b/playbooks/adhoc/upgrades/files/versions.sh index 01ea1d91a..f90719cab 100644 --- a/playbooks/adhoc/upgrades/files/versions.sh +++ b/playbooks/adhoc/upgrades/files/versions.sh @@ -1,7 +1,9 @@ #!/bin/bash -yum_installed=$(yum list installed "$@" | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | awk '{ print $2 }' | tr '\n' ' ') -yum_available=$(yum list available "$@" | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | awk '{ print $2 }' | tr '\n' ' ') +yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') + +yum_available=$(yum list available "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') + echo "---" echo "curr_version: ${yum_installed}" diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index c0fdab027..d854bf913 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -12,10 +12,12 @@ tasks: # Checking the global deployment type rather than host facts, this is about # what the user is requesting. - - fail: msg="Deployment type enterprise not supported for upgrade" + - fail: + msg: "Deployment type enterprise not supported for upgrade" when: deployment_type == "enterprise" # Pacemaker is currently the only supported upgrade path for multiple masters - - fail: msg="openshift_master_cluster_method must be set to 'pacemaker'" + - fail: + msg: "openshift_master_cluster_method must be set to 'pacemaker'" when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker")) - name: Run pre-upgrade checks on first master @@ -119,12 +121,18 @@ - set_fact: g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" - - fail: This playbook requires Origin 1.0.6 or later + - fail: + msg: This playbook requires Origin 1.0.6 or later when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') - - fail: This playbook requires Atomic OpenShift 3.0.2 or later + - fail: + msg: This playbook requires Atomic OpenShift 3.0.2 or later when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2','<') + - fail: + msg: Atomic OpenShift 3.1 packages not found + when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) + - name: Upgrade masters hosts: masters -- cgit v1.2.3 From 8a665c2f3fae20116f0b21bcb7ba644d23ec0376 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 13:41:05 -0500 Subject: Fix removal of kubernetesMasterConfig.apiLevels --- playbooks/adhoc/upgrades/library/openshift_upgrade_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py index f9ee5b254..a6721bb92 100755 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py @@ -75,7 +75,7 @@ def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): changes.append(result['changes']) if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']: - config['kubernetesMasterConfig'].remove('apiLevels') + config['kubernetesMasterConfig'].pop('apiLevels') changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels') # Add proxyClientInfo to master-config -- cgit v1.2.3 From e250f4712eecace09ee37bcfa116206e765d0076 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 13:44:44 -0500 Subject: remove debug statement --- playbooks/adhoc/upgrades/upgrade.yml | 2 -- 1 file changed, 2 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index d854bf913..003e8f397 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -152,8 +152,6 @@ pkg: python-yaml state: installed - - debug: var=hostvars[inventory_hostname].openshift.common.config_base - - name: Upgrade master configuration openshift_upgrade_config: from_version: '3.0' -- cgit v1.2.3 From e1aa8fb073349f44cc9b3f2b6133969b01f63c5b Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Fri, 6 Nov 2015 14:20:52 -0500 Subject: Wait for cluster to recover after pcs resource restart. --- playbooks/adhoc/upgrades/upgrade.yml | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 003e8f397..02847eab3 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -323,6 +323,14 @@ - name: Restart master cluster command: pcs resource restart master when: openshift_master_ha | bool + - name: Wait for the clustered master service to be available + wait_for: + host: "{{ openshift_master_cluster_vip }}" + port: 8443 + state: started + timeout: 180 + delay: 90 + when: openshift_master_ha | bool - name: Upgrade default router and registry hosts: masters[0] -- cgit v1.2.3 From 0cfb416780b9e86c26eafd62ee6b99cd2a46b7cd Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Fri, 6 Nov 2015 14:33:17 -0400 Subject: Minor upgrade improvements. Skip some 3.1 checks if doing a 3.0.x to 3.0.2 upgrade. Improve error message when oc whoami fails (i.e. openshift is down) during pre-upgrade checks, rather than assuming the binary doesn't exist. --- playbooks/adhoc/upgrades/files/pre-upgrade-check | 7 +++++-- playbooks/adhoc/upgrades/upgrade.yml | 12 +++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/files/pre-upgrade-check b/playbooks/adhoc/upgrades/files/pre-upgrade-check index c8ecae399..ed4ab6d1b 100644 --- a/playbooks/adhoc/upgrades/files/pre-upgrade-check +++ b/playbooks/adhoc/upgrades/files/pre-upgrade-check @@ -131,8 +131,11 @@ def main(): print "found" except: print( - 'Can not find oc (%s). Override the path with the ' - 'OC_PATH environment variable. Exiting...' % OC_PATH) + 'Unable to run "%s whoami"\n' + 'Please ensure OpenShift is running, and "oc" is on your system ' + 'path.\n' + 'You can override the path with the OC_PATH environment variable.' + % OC_PATH) raise SystemExit(1) # Where the magic happens diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 003e8f397..7377cbd85 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -10,11 +10,6 @@ openshift_master_ha: "{{ groups['masters'] | length > 1 }}" gather_facts: no tasks: - # Checking the global deployment type rather than host facts, this is about - # what the user is requesting. - - fail: - msg: "Deployment type enterprise not supported for upgrade" - when: deployment_type == "enterprise" # Pacemaker is currently the only supported upgrade path for multiple masters - fail: msg: "openshift_master_cluster_method must be set to 'pacemaker'" @@ -132,6 +127,12 @@ - fail: msg: Atomic OpenShift 3.1 packages not found when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) + # Deployment type 'enterprise' is no longer valid if we're upgrading to 3.1 or beyond. + # (still valid for 3.0.x to 3.0.y however) Using the global deployment_type here as + # we're checking what was requested by the upgrade, not the current type on the system. + - fail: + msg: "Deployment type enterprise not supported for upgrade" + when: deployment_type == "enterprise" and g_aos_versions.curr_version | version_compare('3.1', '>=') - name: Upgrade masters @@ -158,6 +159,7 @@ to_version: '3.1' role: master config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" + when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=') - set_fact: master_certs_missing: True -- cgit v1.2.3 From 5e62f5590f0c478da8b11b3d41b563b87cc385dd Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 16:32:00 -0500 Subject: Fix indentation on when --- playbooks/adhoc/upgrades/upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 09c472e58..1b6b5757c 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -159,7 +159,7 @@ to_version: '3.1' role: master config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=') + when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=') - set_fact: master_certs_missing: True -- cgit v1.2.3 From eba9772a02c1c228c2720e7c298c30c7dc078049 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Mon, 9 Nov 2015 10:25:45 -0400 Subject: Remove upgrade playbook restriction on 3.0.2. This is blocking 3.0.1 upgrades to 3.1 incorrectly, which is a scenario we should support. --- playbooks/adhoc/upgrades/upgrade.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 1b6b5757c..42fbd3cf8 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -120,10 +120,7 @@ msg: This playbook requires Origin 1.0.6 or later when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') - - fail: - msg: This playbook requires Atomic OpenShift 3.0.2 or later - when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2','<') - + # TODO: This should be specific to the 3.1 upgrade playbook (coming in future refactor), otherwise we are blocking 3.0.1 to 3.0.2 here. - fail: msg: Atomic OpenShift 3.1 packages not found when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) -- cgit v1.2.3 From ca8a5abf71bd885a80f15f3d743eddfc6371af13 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Mon, 9 Nov 2015 13:01:30 -0400 Subject: Read etcd data dir from appropriate config file. Rather than assuming the etcd data dir, we now read if from master-config.yaml if using embedded etcd, otherwise from etcd.conf. Doing so now required use of PyYAML to parse config file when gathering facts. Fixed discrepancy with data_dir fact and openshift-enterprise deployment_type. --- playbooks/adhoc/upgrades/upgrade.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 1b6b5757c..03b9193c0 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -150,7 +150,7 @@ - name: Ensure python-yaml present for config upgrade yum: - pkg: python-yaml + pkg: PyYAML state: installed - name: Upgrade master configuration -- cgit v1.2.3 From fa67d37245d53f4ea7222b0d4d970ede48fee94b Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 10 Nov 2015 09:46:13 -0400 Subject: 1279734: Ensure services are enabled after upgrade. With the openshift to atomic-openshift renames, some services were not enabled after upgrade. Added enabled directives to all service restart lines in the upgrade playbook. --- playbooks/adhoc/upgrades/upgrade.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 7ce2698db..30f911f6f 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -226,7 +226,7 @@ when: inventory_hostname != groups.masters.0 - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted + service: name="{{ openshift.common.service_type}}-master" enabled=yes state=restarted when: not openshift_master_ha | bool - name: Destroy cluster @@ -250,7 +250,7 @@ openshift_master_ha: "{{ groups['masters'] | length > 1 }}" tasks: - name: Start pcsd - service: name=pcsd state=started + service: name=pcsd enabled=yes state=started when: openshift_master_ha | bool - name: Re-create cluster @@ -281,7 +281,7 @@ - name: Upgrade node packages command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" state=restarted + service: name="{{ openshift.common.service_type }}-node" enabled=yes state=restarted - name: Update cluster policy and policy bindings hosts: masters[0] @@ -311,7 +311,7 @@ openshift_master_ha: "{{ groups['masters'] | length > 1 }}" tasks: - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted + service: name="{{ openshift.common.service_type}}-master" enabled=yes state=restarted when: not openshift_master_ha | bool - name: Restart cluster post reconcile -- cgit v1.2.3 From 28d601beeebaacd5e79b3fec336a283c8bf20e68 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 10 Nov 2015 13:39:04 -0400 Subject: Add separate step to enable services during upgrade. Instead of combining this with tasks to restart services, add a separate started+enabled play for masters and nodes at the end of the playbook. --- playbooks/adhoc/upgrades/upgrade.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml index 30f911f6f..324f5fba3 100644 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ b/playbooks/adhoc/upgrades/upgrade.yml @@ -226,7 +226,7 @@ when: inventory_hostname != groups.masters.0 - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" enabled=yes state=restarted + service: name="{{ openshift.common.service_type}}-master" state=restarted when: not openshift_master_ha | bool - name: Destroy cluster @@ -281,7 +281,7 @@ - name: Upgrade node packages command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" enabled=yes state=restarted + service: name="{{ openshift.common.service_type }}-node" state=restarted - name: Update cluster policy and policy bindings hosts: masters[0] @@ -311,7 +311,7 @@ openshift_master_ha: "{{ groups['masters'] | length > 1 }}" tasks: - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" enabled=yes state=restarted + service: name="{{ openshift.common.service_type}}-master" state=restarted when: not openshift_master_ha | bool - name: Restart cluster post reconcile @@ -389,3 +389,19 @@ openshift_deployment_type: "{{ deployment_type }}" roles: - openshift_examples + +- name: Ensure master services enabled + hosts: masters + vars: + openshift_master_ha: "{{ groups['masters'] | length > 1 }}" + tasks: + - name: Enable master services + service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes + when: not openshift_master_ha | bool + +- name: Ensure node services enabled + hosts: nodes + tasks: + - name: Restart node services + service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes + -- cgit v1.2.3 From 2a7cb7936318c4cf8a94d36748f7356349234ded Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Tue, 10 Nov 2015 16:22:20 -0500 Subject: Add origin-clients to uninstall playbook. --- playbooks/adhoc/uninstall.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml index e05ab43f8..e0dbad900 100644 --- a/playbooks/adhoc/uninstall.yml +++ b/playbooks/adhoc/uninstall.yml @@ -59,8 +59,8 @@ - atomic-openshift-master - atomic-openshift-node - atomic-openshift-sdn-ovs - - etcd - corosync + - etcd - openshift - openshift-master - openshift-node @@ -68,6 +68,7 @@ - openshift-sdn-ovs - openvswitch - origin + - origin-clients - origin-master - origin-node - origin-sdn-ovs -- cgit v1.2.3 From 4c1b0dd4ab8f3a5d4fcfa4ba1501ed374793e77a Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 6 Nov 2015 16:56:37 -0500 Subject: Refactor upgrade playbook(s) - Split playbooks into two, one for 3.0 minor upgrades and one for 3.0 to 3.1 upgrades - Move upgrade playbooks to common/openshift/cluster/upgrades from adhoc - Added a byo wrapper playbooks to set the groups based on the byo conventions, other providers will need similar playbooks added eventually - installer wrapper updates for refactored upgrade playbooks - call new 3.0 to 3.1 upgrade playbook - various fixes for edge cases I hit with a really old config laying around. - fix output of host facts to show connect_to value. --- playbooks/adhoc/upgrades/README.md | 21 -- playbooks/adhoc/upgrades/files/pre-upgrade-check | 188 ---------- playbooks/adhoc/upgrades/files/versions.sh | 10 - playbooks/adhoc/upgrades/filter_plugins | 1 - .../upgrades/library/openshift_upgrade_config.py | 154 -------- playbooks/adhoc/upgrades/lookup_plugins | 1 - playbooks/adhoc/upgrades/roles | 1 - playbooks/adhoc/upgrades/upgrade.yml | 407 --------------------- 8 files changed, 783 deletions(-) delete mode 100644 playbooks/adhoc/upgrades/README.md delete mode 100644 playbooks/adhoc/upgrades/files/pre-upgrade-check delete mode 100644 playbooks/adhoc/upgrades/files/versions.sh delete mode 120000 playbooks/adhoc/upgrades/filter_plugins delete mode 100755 playbooks/adhoc/upgrades/library/openshift_upgrade_config.py delete mode 120000 playbooks/adhoc/upgrades/lookup_plugins delete mode 120000 playbooks/adhoc/upgrades/roles delete mode 100644 playbooks/adhoc/upgrades/upgrade.yml (limited to 'playbooks/adhoc') diff --git a/playbooks/adhoc/upgrades/README.md b/playbooks/adhoc/upgrades/README.md deleted file mode 100644 index 6de8a970f..000000000 --- a/playbooks/adhoc/upgrades/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# [NOTE] -This playbook will re-run installation steps overwriting any local -modifications. You should ensure that your inventory has been updated with any -modifications you've made after your initial installation. If you find any items -that cannot be configured via ansible please open an issue at -https://github.com/openshift/openshift-ansible - -# Overview -This playbook is available as a technical preview. It currently performs the -following steps. - - * Upgrade and restart master services - * Upgrade and restart node services - * Applies latest configuration by re-running the installation playbook - * Applies the latest cluster policies - * Updates the default router if one exists - * Updates the default registry if one exists - * Updates image streams and quickstarts - -# Usage -ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/adhoc/upgrades/upgrade.yml diff --git a/playbooks/adhoc/upgrades/files/pre-upgrade-check b/playbooks/adhoc/upgrades/files/pre-upgrade-check deleted file mode 100644 index ed4ab6d1b..000000000 --- a/playbooks/adhoc/upgrades/files/pre-upgrade-check +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env python -""" -Pre-upgrade checks that must be run on a master before proceeding with upgrade. -""" -# This is a script not a python module: -# pylint: disable=invalid-name - -# NOTE: This script should not require any python libs other than what is -# in the standard library. - -__license__ = "ASL 2.0" - -import json -import os -import subprocess -import re - -# The maximum length of container.ports.name -ALLOWED_LENGTH = 15 -# The valid structure of container.ports.name -ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$') -AT_LEAST_ONE_LETTER = re.compile('[a-z]') -# look at OS_PATH for the full path. Default ot 'oc' -OC_PATH = os.getenv('OC_PATH', 'oc') - - -def validate(value): - """ - validate verifies that value matches required conventions - - Rules of container.ports.name validation: - - * must be less that 16 chars - * at least one letter - * only a-z0-9- - * hyphens can not be leading or trailing or next to each other - - :Parameters: - - `value`: Value to validate - """ - if len(value) > ALLOWED_LENGTH: - return False - - if '--' in value: - return False - - # We search since it can be anywhere - if not AT_LEAST_ONE_LETTER.search(value): - return False - - # We match because it must start at the beginning - if not ALLOWED_CHARS.match(value): - return False - return True - - -def list_items(kind): - """ - list_items returns a list of items from the api - - :Parameters: - - `kind`: Kind of item to access - """ - response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind]) - items = json.loads(response) - return items.get("items", []) - - -def get(obj, *paths): - """ - Gets an object - - :Parameters: - - `obj`: A dictionary structure - - `path`: All other non-keyword arguments - """ - ret_obj = obj - for path in paths: - if ret_obj.get(path, None) is None: - return [] - ret_obj = ret_obj[path] - return ret_obj - - -# pylint: disable=too-many-arguments -def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid): - """ - Prints out results in human friendly way. - - :Parameters: - - `namespace`: Namespace of the resource - - `kind`: Kind of the resource - - `item_name`: Name of the resource - - `container_name`: Name of the container. May be "" when kind=Service. - - `port_name`: Name of the port - - `valid`: True if the port is valid - """ - if not valid: - if len(container_name) > 0: - print('%s/%s -n %s (Container="%s" Port="%s")' % ( - kind, item_name, namespace, container_name, port_name)) - else: - print('%s/%s -n %s (Port="%s")' % ( - kind, item_name, namespace, port_name)) - - -def print_validation_header(): - """ - Prints the error header. Should run on the first error to avoid - overwhelming the user. - """ - print """\ -At least one port name does not validate. Valid port names: - - * must be less that 16 chars - * have at least one letter - * only a-z0-9- - * do not start or end with - - * Dashes may not be next to eachother ('--') -""" - - -def main(): - """ - main is the main entry point to this script - """ - try: - # the comma at the end suppresses the newline - print "Checking for oc ...", - subprocess.check_output([OC_PATH, 'whoami']) - print "found" - except: - print( - 'Unable to run "%s whoami"\n' - 'Please ensure OpenShift is running, and "oc" is on your system ' - 'path.\n' - 'You can override the path with the OC_PATH environment variable.' - % OC_PATH) - raise SystemExit(1) - - # Where the magic happens - first_error = True - for kind, path in [ - ('replicationcontrollers', ("spec", "template", "spec", "containers")), - ('pods', ("spec", "containers")), - ('deploymentconfigs', ("spec", "template", "spec", "containers"))]: - for item in list_items(kind): - namespace = item["metadata"]["namespace"] - item_name = item["metadata"]["name"] - for container in get(item, *path): - container_name = container["name"] - for port in get(container, "ports"): - port_name = port.get("name", None) - if not port_name: - # Unnamed ports are OK - continue - valid = validate(port_name) - if not valid and first_error: - first_error = False - print_validation_header() - pretty_print_errors( - namespace, kind, item_name, - container_name, port_name, valid) - - # Services follow a different flow - for item in list_items('services'): - namespace = item["metadata"]["namespace"] - item_name = item["metadata"]["name"] - for port in get(item, "spec", "ports"): - port_name = port.get("targetPort", None) - if isinstance(port_name, int) or port_name is None: - # Integer only or unnamed ports are OK - continue - valid = validate(port_name) - if not valid and first_error: - first_error = False - print_validation_header() - pretty_print_errors( - namespace, "services", item_name, "", port_name, valid) - - # If we had at least 1 error then exit with 1 - if not first_error: - raise SystemExit(1) - - -if __name__ == '__main__': - main() - diff --git a/playbooks/adhoc/upgrades/files/versions.sh b/playbooks/adhoc/upgrades/files/versions.sh deleted file mode 100644 index f90719cab..000000000 --- a/playbooks/adhoc/upgrades/files/versions.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') - -yum_available=$(yum list available "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ') - - -echo "---" -echo "curr_version: ${yum_installed}" -echo "avail_version: ${yum_available}" diff --git a/playbooks/adhoc/upgrades/filter_plugins b/playbooks/adhoc/upgrades/filter_plugins deleted file mode 120000 index b0b7a3414..000000000 --- a/playbooks/adhoc/upgrades/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../filter_plugins/ \ No newline at end of file diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py deleted file mode 100755 index a6721bb92..000000000 --- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# vim: expandtab:tabstop=4:shiftwidth=4 - -"""Ansible module for modifying OpenShift configs during an upgrade""" - -import os -import yaml - -DOCUMENTATION = ''' ---- -module: openshift_upgrade_config -short_description: OpenShift Upgrade Config -author: Jason DeTiberus -requirements: [ ] -''' -EXAMPLES = ''' -''' - -def modify_api_levels(level_list, remove, ensure, msg_prepend='', - msg_append=''): - """ modify_api_levels """ - changed = False - changes = [] - - if not isinstance(remove, list): - remove = [] - - if not isinstance(ensure, list): - ensure = [] - - if not isinstance(level_list, list): - new_list = [] - changed = True - changes.append("%s created missing %s" % (msg_prepend, msg_append)) - else: - new_list = level_list - for level in remove: - if level in new_list: - new_list.remove(level) - changed = True - changes.append("%s removed %s %s" % (msg_prepend, level, msg_append)) - - for level in ensure: - if level not in new_list: - new_list.append(level) - changed = True - changes.append("%s added %s %s" % (msg_prepend, level, msg_append)) - - return {'new_list': new_list, 'changed': changed, 'changes': changes} - - -def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup): - """Main upgrade method for 3.0 to 3.1.""" - changes = [] - - # Facts do not get transferred to the hosts where custom modules run, - # need to make some assumptions here. - master_config = os.path.join(config_base, 'master/master-config.yaml') - - master_cfg_file = open(master_config, 'r') - config = yaml.safe_load(master_cfg_file.read()) - master_cfg_file.close() - - - # Remove unsupported api versions and ensure supported api versions from - # master config - unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3'] - supported_levels = ['v1'] - - result = modify_api_levels(config.get('apiLevels'), unsupported_levels, - supported_levels, 'master-config.yaml:', 'from apiLevels') - if result['changed']: - config['apiLevels'] = result['new_list'] - changes.append(result['changes']) - - if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']: - config['kubernetesMasterConfig'].pop('apiLevels') - changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels') - - # Add proxyClientInfo to master-config - if 'proxyClientInfo' not in config['kubernetesMasterConfig']: - config['kubernetesMasterConfig']['proxyClientInfo'] = { - 'certFile': 'master.proxy-client.crt', - 'keyFile': 'master.proxy-client.key' - } - changes.append("master-config.yaml: added proxyClientInfo") - - if len(changes) > 0: - if backup: - # TODO: Check success: - ansible_module.backup_local(master_config) - - # Write the modified config: - out_file = open(master_config, 'w') - out_file.write(yaml.safe_dump(config, default_flow_style=False)) - out_file.close() - - return changes - - -def upgrade_master(ansible_module, config_base, from_version, to_version, backup): - """Upgrade entry point.""" - if from_version == '3.0': - if to_version == '3.1': - return upgrade_master_3_0_to_3_1(ansible_module, config_base, backup) - - -def main(): - """ main """ - # disabling pylint errors for global-variable-undefined and invalid-name - # for 'global module' usage, since it is required to use ansible_facts - # pylint: disable=global-variable-undefined, invalid-name, - # redefined-outer-name - global module - - module = AnsibleModule( - argument_spec=dict( - config_base=dict(required=True), - from_version=dict(required=True, choices=['3.0']), - to_version=dict(required=True, choices=['3.1']), - role=dict(required=True, choices=['master']), - backup=dict(required=False, default=True, type='bool') - ), - supports_check_mode=True, - ) - - from_version = module.params['from_version'] - to_version = module.params['to_version'] - role = module.params['role'] - backup = module.params['backup'] - config_base = module.params['config_base'] - - try: - changes = [] - if role == 'master': - changes = upgrade_master(module, config_base, from_version, - to_version, backup) - - changed = len(changes) > 0 - return module.exit_json(changed=changed, changes=changes) - - # ignore broad-except error to avoid stack trace to ansible user - # pylint: disable=broad-except - except Exception, e: - return module.fail_json(msg=str(e)) - -# ignore pylint errors related to the module_utils import -# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import -# import module snippets -from ansible.module_utils.basic import * - -if __name__ == '__main__': - main() diff --git a/playbooks/adhoc/upgrades/lookup_plugins b/playbooks/adhoc/upgrades/lookup_plugins deleted file mode 120000 index 73cafffe5..000000000 --- a/playbooks/adhoc/upgrades/lookup_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../lookup_plugins/ \ No newline at end of file diff --git a/playbooks/adhoc/upgrades/roles b/playbooks/adhoc/upgrades/roles deleted file mode 120000 index e2b799b9d..000000000 --- a/playbooks/adhoc/upgrades/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles/ \ No newline at end of file diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml deleted file mode 100644 index 324f5fba3..000000000 --- a/playbooks/adhoc/upgrades/upgrade.yml +++ /dev/null @@ -1,407 +0,0 @@ ---- -- name: Load master facts - hosts: masters - roles: - - openshift_facts - -- name: Verify upgrade can proceed - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - gather_facts: no - tasks: - # Pacemaker is currently the only supported upgrade path for multiple masters - - fail: - msg: "openshift_master_cluster_method must be set to 'pacemaker'" - when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker")) - -- name: Run pre-upgrade checks on first master - hosts: masters[0] - tasks: - # If this script errors out ansible will show the default stdout/stderr - # which contains details for the user: - - script: files/pre-upgrade-check - -- name: Evaluate etcd_hosts - hosts: localhost - tasks: - - name: Evaluate etcd hosts - add_host: - name: "{{ groups.masters.0 }}" - groups: etcd_hosts - when: hostvars[groups.masters.0].openshift.master.embedded_etcd | bool - - name: Evaluate etcd hosts - add_host: - name: "{{ item }}" - groups: etcd_hosts - with_items: groups.etcd - when: not hostvars[groups.masters.0].openshift.master.embedded_etcd | bool - -- name: Backup etcd - hosts: etcd_hosts - vars: - embedded_etcd: "{{ openshift.master.embedded_etcd }}" - timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" - roles: - - openshift_facts - tasks: - - - stat: path=/var/lib/openshift - register: var_lib_openshift - - - stat: path=/var/lib/origin - register: var_lib_origin - - - name: Create origin symlink if necessary - file: src=/var/lib/openshift/ dest=/var/lib/origin state=link - when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False - - - name: Check available disk space for etcd backup - # We assume to be using the data dir for all backups. - shell: > - df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1 - register: avail_disk - - - name: Check current embedded etcd disk usage - shell: > - du -k {{ openshift.master.etcd_data_dir }} | tail -n 1 | cut -f1 - register: etcd_disk_usage - when: embedded_etcd | bool - - - name: Abort if insufficient disk space for etcd backup - fail: - msg: > - {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup, - {{ avail_disk.stdout }} Kb available. - when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int) - - - name: Install etcd (for etcdctl) - yum: - pkg: etcd - state: latest - - - name: Generate etcd backup - command: > - etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }} - --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }} - - - name: Display location of etcd backup - debug: - msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}" - -- name: Update deployment type - hosts: OSEv3 - roles: - - openshift_facts - post_tasks: - - openshift_facts: - role: common - local_facts: - deployment_type: "{{ deployment_type }}" - - -- name: Perform upgrade version checking - hosts: masters[0] - tasks: - - name: Clean yum cache - command: yum clean all - - - name: Determine available versions - script: files/versions.sh {{ openshift.common.service_type }} openshift - register: g_versions_result - - - set_fact: - g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}" - - - set_fact: - g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}" - - - fail: - msg: This playbook requires Origin 1.0.6 or later - when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<') - - # TODO: This should be specific to the 3.1 upgrade playbook (coming in future refactor), otherwise we are blocking 3.0.1 to 3.0.2 here. - - fail: - msg: Atomic OpenShift 3.1 packages not found - when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<')) - # Deployment type 'enterprise' is no longer valid if we're upgrading to 3.1 or beyond. - # (still valid for 3.0.x to 3.0.y however) Using the global deployment_type here as - # we're checking what was requested by the upgrade, not the current type on the system. - - fail: - msg: "Deployment type enterprise not supported for upgrade" - when: deployment_type == "enterprise" and g_aos_versions.curr_version | version_compare('3.1', '>=') - - -- name: Upgrade masters - hosts: masters - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - tasks: - - name: Upgrade to latest available kernel - yum: - pkg: kernel - state: latest - - - name: Upgrade master packages - command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }} - - - name: Ensure python-yaml present for config upgrade - yum: - pkg: PyYAML - state: installed - - - name: Upgrade master configuration - openshift_upgrade_config: - from_version: '3.0' - to_version: '3.1' - role: master - config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}" - when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=') - - - set_fact: - master_certs_missing: True - master_cert_subdir: master-{{ openshift.common.hostname }} - master_cert_config_dir: "{{ openshift.common.config_base }}/master" - -- name: Create temp directory for syncing certs - hosts: localhost - gather_facts: no - tasks: - - name: Create local temp directory for syncing certs - local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX - register: g_master_mktemp - changed_when: False - -- name: Generate missing master certificates - hosts: masters[0] - vars: - master_hostnames: "{{ hostvars - | oo_select_keys(groups.masters) - | oo_collect('openshift.common.all_hostnames') - | oo_flatten | unique }}" - master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs" - masters_needing_certs: "{{ hostvars - | oo_select_keys(groups.masters) - | difference([groups.masters.0]) }}" - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - openshift_deployment_type: "{{ deployment_type }}" - roles: - - openshift_master_certificates - post_tasks: - - name: Remove generated etcd client certs when using external etcd - file: - path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" - state: absent - when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config - with_nested: - - masters_needing_certs - - - master.etcd-client.crt - - master.etcd-client.key - - - name: Create a tarball of the master certs - command: > - tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz - -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . - with_items: masters_needing_certs - - - name: Retrieve the master cert tarball from the master - fetch: - src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" - dest: "{{ sync_tmpdir }}/" - flat: yes - fail_on_missing: yes - validate_checksum: yes - with_items: masters_needing_certs - -- name: Sync certs and restart masters post configuration change - hosts: masters - vars: - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Unarchive the tarball on the master - unarchive: - src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz" - dest: "{{ master_cert_config_dir }}" - when: inventory_hostname != groups.masters.0 - - - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted - when: not openshift_master_ha | bool - -- name: Destroy cluster - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - openshift_deployment_type: "{{ deployment_type }}" - pre_tasks: - - name: Check for configured cluster - stat: - path: /etc/corosync/corosync.conf - register: corosync_conf - when: openshift_master_ha | bool - - name: Destroy cluster - command: pcs cluster destroy --all - when: openshift_master_ha | bool and corosync_conf.stat.exists == true - -- name: Start pcsd on masters - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Start pcsd - service: name=pcsd enabled=yes state=started - when: openshift_master_ha | bool - -- name: Re-create cluster - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - openshift_deployment_type: "{{ deployment_type }}" - omc_cluster_hosts: "{{ groups.masters | join(' ') }}" - roles: - - role: openshift_master_cluster - when: openshift_master_ha | bool - -- name: Delete temporary directory on localhost - hosts: localhost - gather_facts: no - tasks: - - file: name={{ g_master_mktemp.stdout }} state=absent - changed_when: False - - -- name: Upgrade nodes - hosts: nodes - vars: - openshift_version: "{{ openshift_pkg_version | default('') }}" - roles: - - openshift_facts - tasks: - - name: Upgrade node packages - command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }} - - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" state=restarted - -- name: Update cluster policy and policy bindings - hosts: masters[0] - vars: - origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}" - ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version | version_compare('3.0.2','>') }}" - tasks: - - name: oadm policy reconcile-cluster-roles --confirm - command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig - policy reconcile-cluster-roles --confirm - - - name: oadm policy reconcile-cluster-role-bindings --confirm - command: > - {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig - policy reconcile-cluster-role-bindings - --exclude-groups=system:authenticated - --exclude-groups=system:unauthenticated - --exclude-users=system:anonymous - --additive-only=true --confirm - when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool - - -- name: Restart masters post reconcile - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Restart master services - service: name="{{ openshift.common.service_type}}-master" state=restarted - when: not openshift_master_ha | bool - -- name: Restart cluster post reconcile - hosts: masters[0] - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Restart master cluster - command: pcs resource restart master - when: openshift_master_ha | bool - - name: Wait for the clustered master service to be available - wait_for: - host: "{{ openshift_master_cluster_vip }}" - port: 8443 - state: started - timeout: 180 - delay: 90 - when: openshift_master_ha | bool - -- name: Upgrade default router and registry - hosts: masters[0] - vars: - - registry_image: "{{ openshift.master.registry_url | replace( '${component}', 'docker-registry' ) | replace ( '${version}', 'v' + g_new_version ) }}" - - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}" - - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig" - tasks: - - name: Check for default router - command: > - {{ oc_cmd }} get -n default dc/router - register: _default_router - failed_when: false - changed_when: false - - name: Check for allowHostNetwork and allowHostPorts - when: _default_router.rc == 0 - shell: > - {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork - register: _scc - - name: Grant allowHostNetwork and allowHostPorts - when: - - _default_router.rc == 0 - - "'false' in _scc.stdout" - command: > - {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9 - - name: Update deployment config to 1.0.4/3.0.1 spec - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p - '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}' - - name: Switch to hostNetwork=true - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}' - - name: Update router image to current version - when: _default_router.rc == 0 - command: > - {{ oc_cmd }} patch dc/router -p - '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}' - - - name: Check for default registry - command: > - {{ oc_cmd }} get -n default dc/docker-registry - register: _default_registry - failed_when: false - changed_when: false - - name: Update registry image to current version - when: _default_registry.rc == 0 - command: > - {{ oc_cmd }} patch dc/docker-registry -p - '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}' - -- name: Update image streams and templates - hosts: masters[0] - vars: - openshift_examples_import_command: "update" - openshift_deployment_type: "{{ deployment_type }}" - roles: - - openshift_examples - -- name: Ensure master services enabled - hosts: masters - vars: - openshift_master_ha: "{{ groups['masters'] | length > 1 }}" - tasks: - - name: Enable master services - service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes - when: not openshift_master_ha | bool - -- name: Ensure node services enabled - hosts: nodes - tasks: - - name: Restart node services - service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes - -- cgit v1.2.3