From 6bc3a3ac71e11fb6459df715536fec373c123a97 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Wed, 7 Mar 2018 07:03:57 +0100 Subject: Streamlined networking, OpenShift recovery, Ganesha --- roles/glusterfs/defaults/main.yml | 2 +- roles/glusterfs/tasks/common.yml | 1 + roles/glusterfs/tasks/create_domain.yml | 8 ++++ roles/glusterfs/tasks/ganesha.yml | 6 +++ roles/glusterfs/tasks/install.yml | 34 +++++++++++++++++ roles/glusterfs/tasks/iterate_domains.yml | 7 ++++ roles/glusterfs/tasks/iterate_volumes.yml | 12 ++++++ roles/glusterfs/tasks/main.yml | 35 +---------------- roles/glusterfs/tasks/migrate.yml | 34 +++++++++++++++++ roles/glusterfs/tasks/migrate_failed_brick.yml | 10 +++++ roles/glusterfs/tasks/migrate_inform.yml | 1 + roles/glusterfs/tasks/migrate_volume.yml | 17 +++++++++ roles/glusterfs/tasks/volumes.yml | 2 +- roles/glusterfs/templates/export.openshift.conf.j2 | 44 ++++++++++++++++++++++ 14 files changed, 178 insertions(+), 35 deletions(-) create mode 100644 roles/glusterfs/tasks/ganesha.yml create mode 100644 roles/glusterfs/tasks/install.yml create mode 100644 roles/glusterfs/tasks/iterate_domains.yml create mode 100644 roles/glusterfs/tasks/iterate_volumes.yml create mode 100644 roles/glusterfs/tasks/migrate.yml create mode 100644 roles/glusterfs/tasks/migrate_failed_brick.yml create mode 100644 roles/glusterfs/tasks/migrate_inform.yml create mode 100644 roles/glusterfs/tasks/migrate_volume.yml create mode 100644 roles/glusterfs/templates/export.openshift.conf.j2 (limited to 'roles/glusterfs') diff --git a/roles/glusterfs/defaults/main.yml b/roles/glusterfs/defaults/main.yml index 9587a9b..700838d 100644 --- a/roles/glusterfs/defaults/main.yml +++ b/roles/glusterfs/defaults/main.yml @@ -1,5 +1,5 @@ --- -glusterfs_version: 39 +glusterfs_version: 312 glusterfs_transport: rdma glusterfs_network: "{{ ands_storage_network }}" diff --git a/roles/glusterfs/tasks/common.yml b/roles/glusterfs/tasks/common.yml index 5e8e3b6..67fb815 100644 --- a/roles/glusterfs/tasks/common.yml +++ b/roles/glusterfs/tasks/common.yml @@ -8,6 +8,7 @@ - glusterfs-cli - glusterfs-fuse - glusterfs-rdma + - heketi-client - libsemanage-python - name: Allow fuse in SELinux configuration diff --git a/roles/glusterfs/tasks/create_domain.yml b/roles/glusterfs/tasks/create_domain.yml index 8f8042b..76623f2 100644 --- a/roles/glusterfs/tasks/create_domain.yml +++ b/roles/glusterfs/tasks/create_domain.yml @@ -1,8 +1,16 @@ --- +- name: Get list of existing gluster volumes + shell: "gluster volume info" + changed_when: false + register: gv_results + + - name: Configure volumes include_tasks: create_volume.yml with_dict: "{{ domain.volumes }}" + when: volume_string not in gv_results.stdout_lines vars: + volume_string: "Volume Name: {{ volume.key }}" domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}" loop_control: loop_var: volume diff --git a/roles/glusterfs/tasks/ganesha.yml b/roles/glusterfs/tasks/ganesha.yml new file mode 100644 index 0000000..61d151a --- /dev/null +++ b/roles/glusterfs/tasks/ganesha.yml @@ -0,0 +1,6 @@ +- name: Create /etc/ganesha/exports + file: dest="/etc/ganesha/exports" owner="root" group="root" mode="0755" state="directory" + +- name: Configure Ganesha NFS exports + template: src="export.openshift.conf.j2" dest="/etc/ganesha/exports/export.openshift.conf" owner=root group=root mode="0644" + diff --git a/roles/glusterfs/tasks/install.yml b/roles/glusterfs/tasks/install.yml new file mode 100644 index 0000000..d7ee766 --- /dev/null +++ b/roles/glusterfs/tasks/install.yml @@ -0,0 +1,34 @@ +--- +- name: Install GlusterFS Common Software + include_tasks: common.yml + when: + - "'software' in glusterfs_subroles" + +- name: Install GlusterFS client + include_tasks: setup-client.yml + when: + - "'software' in glusterfs_subroles" + - "'ands_storage_servers' not in group_names" + +- name: Install GlusterFS OpenShift Server + include_tasks: setup-openshift-server.yml + when: + - "'software' in glusterfs_subroles" + - "'ands_storage_servers' in group_names" + - "'glusterfs' in group_names" + +- name: Install GlusterFS External Server + include_tasks: setup-external-server.yml + when: + - "'software' in glusterfs_subroles" + - "'ands_storage_servers' in group_names" + - "'glusterfs' not in group_names" + +- name: Configure gluster peers (on first host) + shell: gluster peer probe {{item}} + run_once: true + with_items: "{{ glusterfs_servers }}" + +- include_tasks: volumes.yml + when: + - "'volumes' in glusterfs_subroles" diff --git a/roles/glusterfs/tasks/iterate_domains.yml b/roles/glusterfs/tasks/iterate_domains.yml new file mode 100644 index 0000000..e061652 --- /dev/null +++ b/roles/glusterfs/tasks/iterate_domains.yml @@ -0,0 +1,7 @@ +- name: Process all storage domains + include_tasks: "iterate_volumes.yml" + run_once: true + delegate_to: "{{ groups[domain.servers][0] }}" + with_items: "{{ glusterfs_domains }}" + loop_control: + loop_var: domain diff --git a/roles/glusterfs/tasks/iterate_volumes.yml b/roles/glusterfs/tasks/iterate_volumes.yml new file mode 100644 index 0000000..8f61116 --- /dev/null +++ b/roles/glusterfs/tasks/iterate_volumes.yml @@ -0,0 +1,12 @@ +--- +- name: Iterate volumes + include_tasks: "{{ action }}.yml" + with_dict: "{{ domain.volumes }}" + vars: + name: "{{ volume.key }}" + path: "{{ volume.value.mount }}" + server_group: "{{ domain.servers }}" + domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}" + when: volume.value.mount is defined + loop_control: + loop_var: volume diff --git a/roles/glusterfs/tasks/main.yml b/roles/glusterfs/tasks/main.yml index d7ee766..a02c1a1 100644 --- a/roles/glusterfs/tasks/main.yml +++ b/roles/glusterfs/tasks/main.yml @@ -1,34 +1,3 @@ --- -- name: Install GlusterFS Common Software - include_tasks: common.yml - when: - - "'software' in glusterfs_subroles" - -- name: Install GlusterFS client - include_tasks: setup-client.yml - when: - - "'software' in glusterfs_subroles" - - "'ands_storage_servers' not in group_names" - -- name: Install GlusterFS OpenShift Server - include_tasks: setup-openshift-server.yml - when: - - "'software' in glusterfs_subroles" - - "'ands_storage_servers' in group_names" - - "'glusterfs' in group_names" - -- name: Install GlusterFS External Server - include_tasks: setup-external-server.yml - when: - - "'software' in glusterfs_subroles" - - "'ands_storage_servers' in group_names" - - "'glusterfs' not in group_names" - -- name: Configure gluster peers (on first host) - shell: gluster peer probe {{item}} - run_once: true - with_items: "{{ glusterfs_servers }}" - -- include_tasks: volumes.yml - when: - - "'volumes' in glusterfs_subroles" +- name: "Configuring Gluster storage subsystem" + include_tasks: "{{ action | default('install') }}.yml" diff --git a/roles/glusterfs/tasks/migrate.yml b/roles/glusterfs/tasks/migrate.yml new file mode 100644 index 0000000..3ef7917 --- /dev/null +++ b/roles/glusterfs/tasks/migrate.yml @@ -0,0 +1,34 @@ +- debug: msg="Migrating" + +#- name: Detect if source host {{ glusterfs_migrate_from }} is failed or running +# wait_for: host={{ glusterfs_migrate_from }} port=24007 timeout=1 +# register: srcres +# changed_when: srcres | failed +# failed_when: false + +#- set_fact: +# glusterfs_migrate_failed: "{{ srcres | changed }}" + +- set_fact: + glusterfs_migrate_failed: "1" + +- name: Analyze current configuration + include_tasks: iterate_domains.yml + vars: + action: "migrate_volume" + migrate_action: "migrate_inform" + migrate_from: "{{ glusterfs_migrate_from }}" + migrate_to: "{{ glusterfs_migrate_to }}" + migrate_failed: "{{ glusterfs_migrate_failed }}" + +- name: Exterminate mankind + pause: prompt='Please confirm if you want to proceed. Press Ctrl+c and then "a" to abort' + +- name: Analyze current configuration + include_tasks: iterate_domains.yml + vars: + action: "migrate_volume" + migrate_action: "{{ migrate_failed | ternary('migrate_failed_brick', 'migrate_live_brick') }}" + migrate_from: "{{ glusterfs_migrate_from }}" + migrate_to: "{{ glusterfs_migrate_to }}" + migrate_failed: "{{ glusterfs_migrate_failed }}" diff --git a/roles/glusterfs/tasks/migrate_failed_brick.yml b/roles/glusterfs/tasks/migrate_failed_brick.yml new file mode 100644 index 0000000..3490c82 --- /dev/null +++ b/roles/glusterfs/tasks/migrate_failed_brick.yml @@ -0,0 +1,10 @@ +- name: "Volume {{ vol }} - Migrating {{ src }} to {{ dst }}" + command: "gluster volume replace-brick {{ vol }} {{ src }} {{ dst }} commit force" + +# This is only working within containers with 'oc rsh'. But actually there are auto-healing daemons running. +# So we probably find just migrating +#- name: "Healing {{ vol }}" +# command: "gluster volume heal {{ vol }} full" + +#- name: "Rebalancing {{ vol }}" +# command: "gluster volume rebalance {{ vol }} fix-layout start" diff --git a/roles/glusterfs/tasks/migrate_inform.yml b/roles/glusterfs/tasks/migrate_inform.yml new file mode 100644 index 0000000..912d359 --- /dev/null +++ b/roles/glusterfs/tasks/migrate_inform.yml @@ -0,0 +1 @@ +- warn: msg="Volume {{ vol }} - Migrating {{ src }} to {{ dst }}" diff --git a/roles/glusterfs/tasks/migrate_volume.yml b/roles/glusterfs/tasks/migrate_volume.yml new file mode 100644 index 0000000..f9edeac --- /dev/null +++ b/roles/glusterfs/tasks/migrate_volume.yml @@ -0,0 +1,17 @@ +- name: "Analyze bricks of {{ volume.key }}" + shell: | + gluster volume info '{{ volume.key }}' | grep -P 'Brick\d+:' | awk '{ print $2 }' + register: gvires + +- name: "Execute configured {{ migrate_action }} on volume {{ volume.key }} with bricks {{ src }} and {{ dst }}" + include_tasks: "{{ migrate_action }}.yml" + vars: + bricks: "{{ gvires.stdout_lines | list }}" + servers: "{{ bricks | map('regex_replace', ':.*$', '') | list }}" + brick: "{{ servers.index(migrate_from) }}" + src: "{{ bricks[brick | int] | default('') }}" + dst: "{{ bricks[brick | int] | default('') | regex_replace('^' ~ migrate_from, migrate_to) }}" + vol: "{{ volume.key }}" + when: + - migrate_from in servers + - brick is defined diff --git a/roles/glusterfs/tasks/volumes.yml b/roles/glusterfs/tasks/volumes.yml index c4d49ac..1a85378 100644 --- a/roles/glusterfs/tasks/volumes.yml +++ b/roles/glusterfs/tasks/volumes.yml @@ -8,7 +8,7 @@ - name: Mount volume domains include_tasks: mount_domain.yml - when: ( domain.clients | default("---") ) in group_names + when: domain.clients | default([]) | intersect(group_names) | length > 0 with_items: "{{ glusterfs_domains }}" loop_control: loop_var: domain diff --git a/roles/glusterfs/templates/export.openshift.conf.j2 b/roles/glusterfs/templates/export.openshift.conf.j2 new file mode 100644 index 0000000..b2c547f --- /dev/null +++ b/roles/glusterfs/templates/export.openshift.conf.j2 @@ -0,0 +1,44 @@ +{% set i = 0 %} +{% for domain in glusterfs_domains %} +{% for name, vol in domain.volumes.iteritems() %} +{% if vol.nfs_clients is defined %} +{% set nfs = vol.nfs_clients %} +{% set i = i + 1 %} +EXPORT { + Export_Id = {{ i }}; + Path = "/{{ name }}"; + FSAL { + name = GLUSTER; + hostname = "localhost"; + volume = "{{ name }}"; + } + Access_type = RW; + Disable_ACL = true; + Squash = "No_root_squash"; + Pseudo = "/{{ name }}"; + Protocols = "3", "4" ; + Transports = "UDP","TCP"; + SecType = "sys"; + +{% if nfs.rw is defined %} +{% for net in nfs.rw %} + CLIENT { + clients = {{ net }}; + Access_type = RW; + } +{% endfor %} +{% endif %} + +{% if nfs.ro is defined %} +{% for net in nfs.ro %} + CLIENT { + clients = {{ net }}; + Access_type = RO; + } +{% endfor %} +{% endif %} +} + +{% endif %} +{% endfor %} +{% endfor %} -- cgit v1.2.3