Streamlined networking, OpenShift recovery, Ganesha

14 files changed, 178 insertions, 35 deletions

diff --git a/roles/glusterfs/defaults/main.yml b/roles/glusterfs/defaults/main.yml
index 9587a9b..700838d 100644
--- a/roles/glusterfs/defaults/main.yml
+++ b/roles/glusterfs/defaults/main.yml
@@ -1,5 +1,5 @@
 ---
-glusterfs_version: 39
+glusterfs_version: 312
 glusterfs_transport: rdma
 
 glusterfs_network: "{{ ands_storage_network }}"
diff --git a/roles/glusterfs/tasks/common.yml b/roles/glusterfs/tasks/common.yml
index 5e8e3b6..67fb815 100644
--- a/roles/glusterfs/tasks/common.yml
+++ b/roles/glusterfs/tasks/common.yml
@@ -8,6 +8,7 @@
     - glusterfs-cli
     - glusterfs-fuse
     - glusterfs-rdma
+    - heketi-client
     - libsemanage-python
             
 - name: Allow fuse in SELinux configuration
diff --git a/roles/glusterfs/tasks/create_domain.yml b/roles/glusterfs/tasks/create_domain.yml
index 8f8042b..76623f2 100644
--- a/roles/glusterfs/tasks/create_domain.yml
+++ b/roles/glusterfs/tasks/create_domain.yml
@@ -1,8 +1,16 @@
 ---
+- name: Get list of existing gluster volumes
+  shell: "gluster volume info"
+  changed_when: false
+  register: gv_results
+
+
 - name: Configure volumes
   include_tasks: create_volume.yml
   with_dict: "{{ domain.volumes }}"
+  when: volume_string not in gv_results.stdout_lines
   vars:
+    volume_string: "Volume Name: {{ volume.key }}"
     domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}"
   loop_control:
     loop_var: volume
diff --git a/roles/glusterfs/tasks/ganesha.yml b/roles/glusterfs/tasks/ganesha.yml
new file mode 100644
index 0000000..61d151a
--- /dev/null
+++ b/roles/glusterfs/tasks/ganesha.yml
@@ -0,0 +1,6 @@
+- name: Create /etc/ganesha/exports
+  file: dest="/etc/ganesha/exports" owner="root" group="root" mode="0755" state="directory"
+
+- name: Configure Ganesha NFS exports
+  template: src="export.openshift.conf.j2" dest="/etc/ganesha/exports/export.openshift.conf" owner=root group=root mode="0644"
+
diff --git a/roles/glusterfs/tasks/install.yml b/roles/glusterfs/tasks/install.yml
new file mode 100644
index 0000000..d7ee766
--- /dev/null
+++ b/roles/glusterfs/tasks/install.yml
@@ -0,0 +1,34 @@
+---
+- name: Install GlusterFS Common Software
+  include_tasks: common.yml
+  when:
+    - "'software' in glusterfs_subroles"
+
+- name: Install GlusterFS client
+  include_tasks: setup-client.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' not in group_names"
+
+- name: Install GlusterFS OpenShift Server
+  include_tasks: setup-openshift-server.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' in group_names"
+    - "'glusterfs' in group_names"
+
+- name: Install GlusterFS External Server
+  include_tasks: setup-external-server.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' in group_names"
+    - "'glusterfs' not in group_names"
+
+- name: Configure gluster peers (on first host)
+  shell: gluster peer probe {{item}}
+  run_once: true
+  with_items: "{{ glusterfs_servers }}"
+
+- include_tasks: volumes.yml
+  when:
+    - "'volumes' in glusterfs_subroles"
diff --git a/roles/glusterfs/tasks/iterate_domains.yml b/roles/glusterfs/tasks/iterate_domains.yml
new file mode 100644
index 0000000..e061652
--- /dev/null
+++ b/roles/glusterfs/tasks/iterate_domains.yml
@@ -0,0 +1,7 @@
+- name: Process all storage domains
+  include_tasks: "iterate_volumes.yml"
+  run_once: true
+  delegate_to: "{{ groups[domain.servers][0] }}"
+  with_items: "{{ glusterfs_domains }}"
+  loop_control:
+    loop_var: domain  
diff --git a/roles/glusterfs/tasks/iterate_volumes.yml b/roles/glusterfs/tasks/iterate_volumes.yml
new file mode 100644
index 0000000..8f61116
--- /dev/null
+++ b/roles/glusterfs/tasks/iterate_volumes.yml
@@ -0,0 +1,12 @@
+---
+- name: Iterate volumes
+  include_tasks: "{{ action }}.yml"
+  with_dict: "{{ domain.volumes }}"
+  vars:
+    name: "{{ volume.key }}"
+    path: "{{ volume.value.mount }}"
+    server_group: "{{ domain.servers }}"
+    domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}"
+  when: volume.value.mount is defined
+  loop_control:
+    loop_var: volume
diff --git a/roles/glusterfs/tasks/main.yml b/roles/glusterfs/tasks/main.yml
index d7ee766..a02c1a1 100644
--- a/roles/glusterfs/tasks/main.yml
+++ b/roles/glusterfs/tasks/main.yml
@@ -1,34 +1,3 @@
 ---
-- name: Install GlusterFS Common Software
-  include_tasks: common.yml
-  when:
-    - "'software' in glusterfs_subroles"
-
-- name: Install GlusterFS client
-  include_tasks: setup-client.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' not in group_names"
-
-- name: Install GlusterFS OpenShift Server
-  include_tasks: setup-openshift-server.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' in group_names"
-    - "'glusterfs' in group_names"
-
-- name: Install GlusterFS External Server
-  include_tasks: setup-external-server.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' in group_names"
-    - "'glusterfs' not in group_names"
-
-- name: Configure gluster peers (on first host)
-  shell: gluster peer probe {{item}}
-  run_once: true
-  with_items: "{{ glusterfs_servers }}"
-
-- include_tasks: volumes.yml
-  when:
-    - "'volumes' in glusterfs_subroles"
+- name: "Configuring Gluster storage subsystem"
+  include_tasks: "{{ action | default('install') }}.yml"
diff --git a/roles/glusterfs/tasks/migrate.yml b/roles/glusterfs/tasks/migrate.yml
new file mode 100644
index 0000000..3ef7917
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate.yml
@@ -0,0 +1,34 @@
+- debug: msg="Migrating"
+
+#- name: Detect if source host {{ glusterfs_migrate_from }} is failed or running
+#  wait_for: host={{ glusterfs_migrate_from }} port=24007 timeout=1
+#  register: srcres
+#  changed_when: srcres | failed
+#  failed_when: false
+
+#- set_fact: 
+#    glusterfs_migrate_failed: "{{ srcres | changed }}"
+
+- set_fact: 
+    glusterfs_migrate_failed: "1"
+
+- name: Analyze current configuration
+  include_tasks: iterate_domains.yml
+  vars:
+    action: "migrate_volume"
+    migrate_action: "migrate_inform"
+    migrate_from: "{{ glusterfs_migrate_from }}"
+    migrate_to: "{{ glusterfs_migrate_to }}"
+    migrate_failed: "{{ glusterfs_migrate_failed }}"
+
+- name: Exterminate mankind
+  pause: prompt='Please confirm if you want to proceed. Press Ctrl+c and then "a" to abort'
+
+- name: Analyze current configuration
+  include_tasks: iterate_domains.yml
+  vars:
+    action: "migrate_volume"
+    migrate_action: "{{ migrate_failed | ternary('migrate_failed_brick', 'migrate_live_brick') }}"
+    migrate_from: "{{ glusterfs_migrate_from }}"
+    migrate_to: "{{ glusterfs_migrate_to }}"
+    migrate_failed: "{{ glusterfs_migrate_failed }}"
diff --git a/roles/glusterfs/tasks/migrate_failed_brick.yml b/roles/glusterfs/tasks/migrate_failed_brick.yml
new file mode 100644
index 0000000..3490c82
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_failed_brick.yml
@@ -0,0 +1,10 @@
+- name: "Volume {{ vol }} - Migrating {{ src }} to {{ dst }}"
+  command: "gluster volume replace-brick {{ vol }} {{ src }} {{ dst }} commit force"
+
+# This is only working within containers with 'oc rsh'. But actually there are auto-healing daemons running.
+# So we probably find just migrating
+#- name: "Healing {{ vol }}"
+#  command: "gluster volume heal {{ vol }} full"
+
+#- name: "Rebalancing {{ vol }}"
+#  command: "gluster volume rebalance {{ vol }} fix-layout start"
diff --git a/roles/glusterfs/tasks/migrate_inform.yml b/roles/glusterfs/tasks/migrate_inform.yml
new file mode 100644
index 0000000..912d359
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_inform.yml
@@ -0,0 +1 @@
+- warn: msg="Volume {{ vol }} - Migrating {{ src }} to {{ dst }}"
diff --git a/roles/glusterfs/tasks/migrate_volume.yml b/roles/glusterfs/tasks/migrate_volume.yml
new file mode 100644
index 0000000..f9edeac
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_volume.yml
@@ -0,0 +1,17 @@
+- name: "Analyze bricks of {{ volume.key }}"
+  shell: |
+    gluster volume info '{{ volume.key }}' | grep -P 'Brick\d+:' | awk '{ print $2 }'
+  register: gvires
+
+- name: "Execute configured {{ migrate_action }} on volume {{ volume.key }} with bricks {{ src }} and {{ dst }}"
+  include_tasks: "{{ migrate_action }}.yml"
+  vars:
+    bricks: "{{ gvires.stdout_lines | list }}"
+    servers: "{{ bricks | map('regex_replace', ':.*$', '') | list }}"
+    brick: "{{ servers.index(migrate_from) }}"
+    src: "{{ bricks[brick | int] | default('') }}"
+    dst: "{{ bricks[brick | int] | default('') | regex_replace('^' ~ migrate_from, migrate_to) }}"
+    vol: "{{ volume.key }}"
+  when:
+    - migrate_from in servers
+    - brick is defined
diff --git a/roles/glusterfs/tasks/volumes.yml b/roles/glusterfs/tasks/volumes.yml
index c4d49ac..1a85378 100644
--- a/roles/glusterfs/tasks/volumes.yml
+++ b/roles/glusterfs/tasks/volumes.yml
@@ -8,7 +8,7 @@
 
 - name: Mount volume domains
   include_tasks: mount_domain.yml
-  when: ( domain.clients | default("---") ) in group_names
+  when: domain.clients | default([]) | intersect(group_names) | length > 0
   with_items: "{{ glusterfs_domains }}"
   loop_control:
     loop_var: domain  
diff --git a/roles/glusterfs/templates/export.openshift.conf.j2 b/roles/glusterfs/templates/export.openshift.conf.j2
new file mode 100644
index 0000000..b2c547f
--- /dev/null
+++ b/roles/glusterfs/templates/export.openshift.conf.j2
@@ -0,0 +1,44 @@
+{% set i = 0 %}
+{% for domain in glusterfs_domains %}
+{%   for name, vol in domain.volumes.iteritems() %}
+{%      if vol.nfs_clients is defined %}
+{%        set nfs = vol.nfs_clients %}
+{%        set i = i + 1 %}
+EXPORT {
+      Export_Id = {{ i }};
+      Path = "/{{ name }}";
+      FSAL {
+            name = GLUSTER;
+            hostname = "localhost";
+            volume = "{{ name }}";
+      }
+      Access_type = RW;
+      Disable_ACL = true;
+      Squash = "No_root_squash";
+      Pseudo = "/{{ name }}";
+      Protocols = "3", "4" ;
+      Transports = "UDP","TCP";
+      SecType = "sys";
+
+{%        if nfs.rw is defined %}
+{%          for net in nfs.rw %}
+      CLIENT {
+            clients = {{ net }};
+            Access_type = RW;
+      }
+{%          endfor %}
+{%        endif  %}
+
+{%        if nfs.ro is defined %}
+{%          for net in nfs.ro %}
+      CLIENT {
+            clients = {{ net }};
+            Access_type = RO;
+      }
+{%          endfor %}
+{%        endif  %}
+}
+
+{%     endif %}
+{%   endfor %}
+{% endfor %}