diff options
| -rw-r--r-- | inventory/byo/hosts.example | 20 | ||||
| -rw-r--r-- | playbooks/common/openshift-master/config.yml | 29 | ||||
| -rwxr-xr-x | roles/openshift_facts/library/openshift_facts.py | 42 | ||||
| -rw-r--r-- | roles/openshift_master/defaults/main.yml | 6 | ||||
| -rw-r--r-- | roles/openshift_master/handlers/main.yml | 1 | ||||
| -rw-r--r-- | roles/openshift_master/tasks/main.yml | 21 | ||||
| -rw-r--r-- | roles/openshift_master_ca/tasks/main.yml | 2 | ||||
| -rw-r--r-- | roles/openshift_master_certificates/tasks/main.yml | 16 | ||||
| -rw-r--r-- | roles/openshift_master_certificates/vars/main.yml | 3 | ||||
| -rw-r--r-- | roles/openshift_master_cluster/README.md | 34 | ||||
| -rw-r--r-- | roles/openshift_master_cluster/meta/main.yml | 16 | ||||
| -rw-r--r-- | roles/openshift_master_cluster/tasks/configure.yml | 43 | ||||
| -rw-r--r-- | roles/openshift_master_cluster/tasks/configure_deferred.yml | 8 | ||||
| -rw-r--r-- | roles/openshift_master_cluster/tasks/main.yml | 13 | ||||
| -rw-r--r-- | roles/openshift_node_certificates/tasks/main.yml | 2 |
15 files changed, 234 insertions, 22 deletions
diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example index 41216dd89..56f4da5a2 100644 --- a/inventory/byo/hosts.example +++ b/inventory/byo/hosts.example @@ -38,14 +38,30 @@ deployment_type=enterprise # Allow all auth #openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}] +# master cluster ha variables using pacemaker or RHEL HA +#openshift_master_cluster_password=openshift_cluster +#openshift_master_cluster_vip=192.168.133.25 +#openshift_master_cluster_public_vip=192.168.133.25 +#openshift_master_cluster_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com + +# master cluster ha variables when using a different HA solution +# For installation the value of openshift_master_cluster_hostname must resolve +# to the first master defined in the inventory. +# The HA solution must be manually configured after installation and must ensure +# that openshift-master is running on a single master host. +#openshift_master_cluster_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_defer_ha=True + # host group for masters [masters] -ose3-master-ansible.test.example.com +ose3-master[1:3]-ansible.test.example.com [etcd] ose3-etcd[1:3]-ansible.test.example.com # host group for nodes [nodes] -ose3-master-ansible.test.example.com openshift_scheduleable=False +ose3-master[1:3]-ansible.test.example.com openshift_scheduleable=False ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 3956128e1..904ad2dab 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -27,6 +27,9 @@ api_url: "{{ openshift_master_api_url | default(None) }}" api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}" public_api_url: "{{ openshift_master_public_api_url | default(None) }}" + cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" + cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" + cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}" console_path: "{{ openshift_master_console_path | default(None) }}" console_port: "{{ openshift_master_console_port | default(None) }}" console_url: "{{ openshift_master_console_url | default(None) }}" @@ -152,16 +155,26 @@ roles: - openshift_master_certificates post_tasks: + - name: Remove generated etcd client certs when using external etcd + file: + path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: absent + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + with_nested: + - masters_needing_certs + - - master.etcd-client.crt + - master.etcd-client.key + - name: Create a tarball of the master certs command: > - tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz - -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} . + tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz + -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . args: - creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" + creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" with_items: masters_needing_certs - name: Retrieve the master cert tarball from the master fetch: - src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" + src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" dest: "{{ sync_tmpdir }}/" flat: yes fail_on_missing: yes @@ -172,6 +185,7 @@ hosts: oo_masters_to_config vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" pre_tasks: - name: Ensure certificate directory exists file: @@ -192,9 +206,14 @@ group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }} changed_when: False -- name: Deploy OpenShift examples +- name: Additional master configuration hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}" roles: + - role: openshift_master_cluster + when: openshift_master_ha | bool - openshift_examples # Additional instance config for online deployments diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 727861b07..d733639c3 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -367,9 +367,11 @@ def set_url_facts_if_unset(facts): console_path = facts['master']['console_path'] etcd_use_ssl = facts['master']['etcd_use_ssl'] etcd_hosts = facts['master']['etcd_hosts'] - etcd_port = facts['master']['etcd_port'], + etcd_port = facts['master']['etcd_port'] hostname = facts['common']['hostname'] public_hostname = facts['common']['public_hostname'] + cluster_hostname = facts['master'].get('cluster_hostname') + cluster_public_hostname = facts['master'].get('cluster_public_hostname') if 'etcd_urls' not in facts['master']: etcd_urls = [] @@ -384,24 +386,51 @@ def set_url_facts_if_unset(facts): etcd_port)] facts['master']['etcd_urls'] = etcd_urls if 'api_url' not in facts['master']: - facts['master']['api_url'] = format_url(api_use_ssl, hostname, + api_hostname = cluster_hostname if cluster_hostname else hostname + facts['master']['api_url'] = format_url(api_use_ssl, api_hostname, api_port) if 'public_api_url' not in facts['master']: + api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname facts['master']['public_api_url'] = format_url(api_use_ssl, - public_hostname, + api_public_hostname, api_port) if 'console_url' not in facts['master']: + console_hostname = cluster_hostname if cluster_hostname else hostname facts['master']['console_url'] = format_url(console_use_ssl, - hostname, + console_hostname, console_port, console_path) if 'public_console_url' not in facts['master']: + console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname facts['master']['public_console_url'] = format_url(console_use_ssl, - public_hostname, + console_public_hostname, console_port, console_path) return facts +def set_aggregate_facts(facts): + """ Set aggregate facts + + Args: + facts (dict): existing facts + Returns: + dict: the facts dict updated with aggregated facts + """ + all_hostnames = set() + if 'common' in facts: + all_hostnames.add(facts['common']['hostname']) + all_hostnames.add(facts['common']['public_hostname']) + + if 'master' in facts: + if 'cluster_hostname' in facts['master']: + all_hostnames.add(facts['master']['cluster_hostname']) + if 'cluster_public_hostname' in facts['master']: + all_hostnames.add(facts['master']['cluster_public_hostname']) + + facts['common']['all_hostnames'] = list(all_hostnames) + + return facts + def set_sdn_facts_if_unset(facts): """ Set sdn facts if not already present in facts dict @@ -675,6 +704,7 @@ class OpenShiftFacts(object): facts = set_identity_providers_if_unset(facts) facts = set_registry_url_if_unset(facts) facts = set_sdn_facts_if_unset(facts) + facts = set_aggregate_facts(facts) return dict(openshift=facts) def get_defaults(self, roles): @@ -713,7 +743,7 @@ class OpenShiftFacts(object): session_name='ssn', session_secrets_file='', access_token_max_seconds=86400, auth_token_max_seconds=500, - oauth_grant_method='auto') + oauth_grant_method='auto', cluster_defer_ha=False) defaults['master'] = master if 'node' in roles: diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml index 11195e83e..ca8860099 100644 --- a/roles/openshift_master/defaults/main.yml +++ b/roles/openshift_master/defaults/main.yml @@ -15,6 +15,12 @@ os_firewall_allow: port: 24224/tcp - service: Fluentd td-agent udp port: 24224/udp +- service: pcsd + port: 2224/tcp +- service: Corosync UDP + port: 5404/udp +- service: Corosync UDP + port: 5405/udp os_firewall_deny: - service: OpenShift api http port: 8080/tcp diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml index 6fd4dfb51..d57f9a4ea 100644 --- a/roles/openshift_master/handlers/main.yml +++ b/roles/openshift_master/handlers/main.yml @@ -1,3 +1,4 @@ --- - name: restart openshift-master service: name=openshift-master state=restarted + when: not openshift_master_ha diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 02905f32d..bb1689e5f 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -8,6 +8,10 @@ - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods when: openshift_master_oauth_grant_method is defined +- fail: + msg: "openshift_master_cluster_password must be set for multi-master installations" + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined + - name: Install OpenShift Master package yum: pkg=openshift-master state=present register: install_result @@ -16,6 +20,9 @@ openshift_facts: role: master local_facts: + cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" + cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" + cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}" debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}" api_port: "{{ openshift_master_api_port | default(None) }}" api_url: "{{ openshift_master_api_url | default(None) }}" @@ -114,12 +121,26 @@ - name: Start and enable openshift-master service: name=openshift-master enabled=yes state=started + when: not openshift_master_ha register: start_result - name: pause to prevent service restart from interfering with bootstrapping pause: seconds=30 when: start_result | changed +- name: Install cluster packages + yum: pkg=pcs state=present + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool + register: install_result + +- name: Start and enable cluster service + service: name=pcsd enabled=yes state=started + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool + +- name: Set the cluster user password + shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster + when: install_result | changed + - name: Create the OpenShift client config dir(s) file: path: "~{{ item }}/.kube" diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml index 8163ecd7f..03eb7e15f 100644 --- a/roles/openshift_master_ca/tasks/main.yml +++ b/roles/openshift_master_ca/tasks/main.yml @@ -14,7 +14,7 @@ - name: Create the master certificates if they do not already exist command: > {{ openshift.common.admin_binary }} create-master-certs - --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }} + --hostnames={{ openshift.common.all_hostnames | join(',') }} --master={{ openshift.master.api_url }} --public-master={{ openshift.master.public_api_url }} --cert-dir={{ openshift_master_config_dir }} --overwrite=false diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml index b5a3f8e40..297d53bcd 100644 --- a/roles/openshift_master_certificates/tasks/main.yml +++ b/roles/openshift_master_certificates/tasks/main.yml @@ -7,14 +7,20 @@ with_items: masters_needing_certs - file: - src: "{{ openshift_master_ca_cert }}" - dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt" - with_items: masters_needing_certs + src: "{{ openshift_master_config_dir }}/{{ item.1 }}" + dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: hard + with_nested: + - masters_needing_certs + - - ca.crt + - ca.key + - ca.serial.txt + - name: Create the master certificates if they do not already exist command: > {{ openshift.common.admin_binary }} create-master-certs - --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }} + --hostnames={{ item.openshift.common.all_hostnames | join(',') }} --master={{ item.openshift.master.api_url }} --public-master={{ item.openshift.master.public_api_url }} --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }} @@ -22,3 +28,5 @@ args: creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt" with_items: masters_needing_certs + + diff --git a/roles/openshift_master_certificates/vars/main.yml b/roles/openshift_master_certificates/vars/main.yml index 6e577b13b..6214f7918 100644 --- a/roles/openshift_master_certificates/vars/main.yml +++ b/roles/openshift_master_certificates/vars/main.yml @@ -1,6 +1,3 @@ --- openshift_generated_configs_dir: /etc/openshift/generated-configs openshift_master_config_dir: /etc/openshift/master -openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt" -openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key" -openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt" diff --git a/roles/openshift_master_cluster/README.md b/roles/openshift_master_cluster/README.md new file mode 100644 index 000000000..f150981fa --- /dev/null +++ b/roles/openshift_master_cluster/README.md @@ -0,0 +1,34 @@ +OpenShift Master Cluster +======================== + +TODO + +Requirements +------------ + +TODO + +Role Variables +-------------- + +TODO + +Dependencies +------------ + +TODO + +Example Playbook +---------------- + +TODO + +License +------- + +Apache License Version 2.0 + +Author Information +------------------ + +Jason DeTiberus (jdetiber@redhat.com) diff --git a/roles/openshift_master_cluster/meta/main.yml b/roles/openshift_master_cluster/meta/main.yml new file mode 100644 index 000000000..f3236e850 --- /dev/null +++ b/roles/openshift_master_cluster/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + author: Jason DeTiberus + description: + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.8 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud + - system +dependencies: +- { role: openshift_facts } diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml new file mode 100644 index 000000000..3510f9f62 --- /dev/null +++ b/roles/openshift_master_cluster/tasks/configure.yml @@ -0,0 +1,43 @@ +--- +- fail: + msg: This role requires that openshift_master_cluster_vip is set + when: openshift_master_cluster_vip is not defined or not openshift_master_cluster_vip +- fail: + msg: This role requires that openshift_master_cluster_public_vip is set + when: openshift_master_cluster_public_vip is not defined or not openshift_master_cluster_public_vip + +- name: Authenticate to the cluster + command: pcs cluster auth -u hacluster -p {{ openshift_master_cluster_password }} {{ omc_cluster_hosts }} + +- name: Create the cluster + command: pcs cluster setup --name openshift_master {{ omc_cluster_hosts }} + +- name: Start the cluster + command: pcs cluster start --all + +- name: Enable the cluster on all nodes + command: pcs cluster enable --all + +- name: Set default resource stickiness + command: pcs resource defaults resource-stickiness=100 + +- name: Add the cluster VIP resource + command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_vip }} --group openshift-master + +- name: Add the cluster public VIP resource + command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_public_vip }} --group openshift-master + when: openshift_master_cluster_public_vip != openshift_master_cluster_vip + +- name: Add the cluster openshift-master service resource + command: pcs resource create master systemd:openshift-master op start timeout=90s stop timeout=90s --group openshift-master + +- name: Disable stonith + command: pcs property set stonith-enabled=false + +# TODO: handle case where api port is not 8443 +- name: Wait for the clustered master service to be available + wait_for: + host: "{{ openshift_master_cluster_vip }}" + port: 8443 + state: started + timeout: 300 diff --git a/roles/openshift_master_cluster/tasks/configure_deferred.yml b/roles/openshift_master_cluster/tasks/configure_deferred.yml new file mode 100644 index 000000000..a80b6c5b4 --- /dev/null +++ b/roles/openshift_master_cluster/tasks/configure_deferred.yml @@ -0,0 +1,8 @@ +--- +- debug: msg="Deferring config" + +- name: Start and enable openshift-master + service: + name: openshift-master + state: started + enabled: yes diff --git a/roles/openshift_master_cluster/tasks/main.yml b/roles/openshift_master_cluster/tasks/main.yml new file mode 100644 index 000000000..315947183 --- /dev/null +++ b/roles/openshift_master_cluster/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Test if cluster is already configured + command: pcs status + register: pcs_status + changed_when: false + failed_when: false + when: not openshift.master.cluster_defer_ha | bool + +- include: configure.yml + when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr" + +- include: configure_deferred.yml + when: openshift.master.cluster_defer_ha | bool diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml index 64a799dfb..c9f02aaf0 100644 --- a/roles/openshift_node_certificates/tasks/main.yml +++ b/roles/openshift_node_certificates/tasks/main.yml @@ -25,7 +25,7 @@ command: > {{ openshift.common.admin_binary }} create-server-cert --cert=server.crt --key=server.key --overwrite=true - --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname]|unique|join(",") }} + --hostnames={{ openshift.common.all_hostnames |join(",") }} --signer-cert={{ openshift_master_ca_cert }} --signer-key={{ openshift_master_ca_key }} --signer-serial={{ openshift_master_ca_serial }} |