Merge pull request #4484 from jarrpa/glusterfs-fixes-too

Merged by openshift-bot

13 files changed, 82 insertions, 30 deletions

diff --git a/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml b/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
index e6bb196b8..c504bfb80 100644
--- a/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
+++ b/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
@@ -35,7 +35,7 @@
   mount:
     state: mounted
     fstype: glusterfs
-    src: "{{ groups.oo_glusterfs_to_config[0] }}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
+    src: "{% if 'glusterfs_registry' in groups %}{{ groups.glusterfs_registry[0] }}{% else %}{{ groups.glusterfs[0] }}{% endif %}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
     name: "{{ mktemp.stdout }}"
 
 - name: Set registry volume permissions
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md
index 62fc35299..da4e348b4 100644
--- a/roles/openshift_storage_glusterfs/README.md
+++ b/roles/openshift_storage_glusterfs/README.md
@@ -90,7 +90,8 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | openshift_storage_glusterfs_heketi_admin_key     | auto-generated          | String to use as secret key for performing heketi commands as admin
 | openshift_storage_glusterfs_heketi_user_key      | auto-generated          | String to use as secret key for performing heketi commands as user that can only view or modify volumes
 | openshift_storage_glusterfs_heketi_topology_load | True                    | Load the GlusterFS topology information into heketi
-| openshift_storage_glusterfs_heketi_url           | Undefined               | URL for the heketi REST API, dynamically determined in native mode
+| openshift_storage_glusterfs_heketi_url           | Undefined               | When heketi is native, this sets the hostname portion of the final heketi route URL. When heketi is external, this is the full URL to the heketi service.
+| openshift_storage_glusterfs_heketi_port          | 8080                    | TCP port for external heketi service **NOTE:** This has no effect in native mode
 | openshift_storage_glusterfs_heketi_wipe          | False                   | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
 
 Each role variable also has a corresponding variable to optionally configure a
diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml
index 468877e57..4ff56af9e 100644
--- a/roles/openshift_storage_glusterfs/defaults/main.yml
+++ b/roles/openshift_storage_glusterfs/defaults/main.yml
@@ -13,11 +13,12 @@ openshift_storage_glusterfs_heketi_is_missing: True
 openshift_storage_glusterfs_heketi_deploy_is_missing: True
 openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}"
 openshift_storage_glusterfs_heketi_version: 'latest'
-openshift_storage_glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
-openshift_storage_glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_heketi_admin_key: "{{ omit }}"
+openshift_storage_glusterfs_heketi_user_key: "{{ omit }}"
 openshift_storage_glusterfs_heketi_topology_load: True
 openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"
 openshift_storage_glusterfs_heketi_url: "{{ omit }}"
+openshift_storage_glusterfs_heketi_port: 8080
 
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
 openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
@@ -33,8 +34,9 @@ openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_gl
 openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}"
 openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}"
 openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}"
-openshift_storage_glusterfs_registry_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
-openshift_storage_glusterfs_registry_heketi_user_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_registry_heketi_admin_key: "{{ omit }}"
+openshift_storage_glusterfs_registry_heketi_user_key: "{{ omit }}"
 openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
 openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
 openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"
+openshift_storage_glusterfs_registry_heketi_port: 8080
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
index 81b4fa5dc..4434f750c 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
@@ -29,7 +29,7 @@ objects:
 - kind: Route
   apiVersion: v1
   metadata:
-    name: deploy-heketi-${CLUSTER_NAME}
+    name: ${HEKETI_ROUTE}
     labels:
       glusterfs: deploy-heketi-${CLUSTER_NAME}-route
       deploy-heketi: support
@@ -115,14 +115,19 @@ parameters:
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside
   value: default
+- name: HEKETI_ROUTE
+  displayName: heketi route name
+  description: Set the hostname for the route URL
+  value: "heketi-glusterfs"
 - name: IMAGE_NAME
-  displayName: heketi container name
+  displayName: heketi container image name
   required: True
 - name: IMAGE_VERSION
-  displayName: heketi container versiona
+  displayName: heketi container image version
   required: True
 - name: CLUSTER_NAME
   displayName: GlusterFS cluster name
+  description: A unique name to identify this heketi service, useful for running multiple heketi instances
   value: glusterfs
 - name: TOPOLOGY_PATH
   displayName: heketi topology file location
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
index dc3d2250a..8c5e1ded3 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
@@ -125,11 +125,12 @@ parameters:
   description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=<CLUSTER_NAME>-host\'
   value: '{ "glusterfs": "storage-host" }'
 - name: IMAGE_NAME
-  displayName: GlusterFS container name
+  displayName: GlusterFS container image name
   required: True
 - name: IMAGE_VERSION
-  displayName: GlusterFS container versiona
+  displayName: GlusterFS container image version
   required: True
 - name: CLUSTER_NAME
   displayName: GlusterFS cluster name
+  description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances
   value: storage
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
index 1d8f1abdf..e3fa0a9fb 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
@@ -27,7 +27,7 @@ objects:
 - kind: Route
   apiVersion: v1
   metadata:
-    name: heketi-${CLUSTER_NAME}
+    name: ${HEKETI_ROUTE}
     labels:
       glusterfs: heketi-${CLUSTER_NAME}-route
   spec:
@@ -109,12 +109,17 @@ parameters:
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside
   value: default
+- name: HEKETI_ROUTE
+  displayName: heketi route name
+  description: Set the hostname for the route URL
+  value: "heketi-glusterfs"
 - name: IMAGE_NAME
-  displayName: heketi container name
+  displayName: heketi container image name
   required: True
 - name: IMAGE_VERSION
-  displayName: heketi container versiona
+  displayName: heketi container image version
   required: True
 - name: CLUSTER_NAME
   displayName: GlusterFS cluster name
+  description: A unique name to identify this heketi service, useful for running multiple heketi instances
   value: glusterfs
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index 829c1f51b..4406ef28b 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -123,21 +123,32 @@
   when:
   - glusterfs_heketi_topology_load
 
-- include: heketi_deploy_part1.yml
+- name: Generate heketi admin key
+  set_fact:
+    glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
   when:
   - glusterfs_heketi_is_native
-  - glusterfs_heketi_deploy_is_missing
-  - glusterfs_heketi_is_missing
+  - glusterfs_heketi_admin_key is undefined
 
-- name: Set heketi URL
+- name: Generate heketi user key
   set_fact:
-    glusterfs_heketi_url: "localhost:8080"
+    glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}"
+  until: "glusterfs_heketi_user_key != glusterfs_heketi_admin_key"
+  delay: 1
+  retries: 10
+  when:
+  - glusterfs_heketi_is_native
+  - glusterfs_heketi_user_key is undefined
+
+- include: heketi_deploy_part1.yml
   when:
   - glusterfs_heketi_is_native
+  - glusterfs_heketi_deploy_is_missing
+  - glusterfs_heketi_is_missing
 
 - name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
+    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
 
 - name: Verify heketi service
   command: "{{ glusterfs_heketi_client }} cluster list"
@@ -155,21 +166,43 @@
   - glusterfs_heketi_is_native
   - glusterfs_heketi_is_missing
 
-- name: Create heketi user secret
+- name: Create heketi secret
   oc_secret:
     namespace: "{{ glusterfs_namespace }}"
     state: present
-    name: "heketi-{{ glusterfs_name }}-user-secret"
+    name: "heketi-{{ glusterfs_name }}-secret"
     type: "kubernetes.io/glusterfs"
     force: True
     contents:
     - path: key
-      data: "{{ glusterfs_heketi_user_key }}"
+      data: "{{ glusterfs_heketi_admin_key }}"
+  when:
+  - glusterfs_storageclass
+
+- name: Get heketi route
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    kind: route
+    state: list
+    name: "heketi-{{ glusterfs_name }}"
+  register: heketi_route
+  when:
+  - glusterfs_storageclass
+  - glusterfs_heketi_is_native
+
+- name: Determine StorageClass heketi URL
+  set_fact:
+    glusterfs_heketi_route: "{{ heketi_route.results.results[0]['spec']['host'] }}"
+  when:
+  - glusterfs_storageclass
+  - glusterfs_heketi_is_native
 
 - name: Generate GlusterFS StorageClass file
   template:
     src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2"
     dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml"
+  when:
+  - glusterfs_storageclass
 
 - name: Create GlusterFS StorageClass
   oc_obj:
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
index aa303d126..dbfe126a4 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
@@ -19,6 +19,7 @@
     glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}"
+    glusterfs_heketi_port: "{{ openshift_storage_glusterfs_heketi_port }}"
     glusterfs_nodes: "{{ groups.glusterfs }}"
 
 - include: glusterfs_common.yml
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
index 4c6891eeb..0849f2a2e 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -19,12 +19,13 @@
     glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_registry_heketi_topology_load }}"
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}"
-    glusterfs_nodes: "{{ groups.glusterfs_registry }}"
+    glusterfs_heketi_port: "{{ openshift_storage_glusterfs_registry_heketi_port }}"
+    glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}"
 
 - include: glusterfs_common.yml
   when:
-  - groups.glusterfs_registry | default([]) | count > 0
-  - "'glusterfs' not in groups or groups.glusterfs_registry != groups.glusterfs"
+  - glusterfs_nodes | default([]) | count > 0
+  - "'glusterfs' not in groups or glusterfs_nodes != groups.glusterfs"
 
 - name: Delete pre-existing GlusterFS registry resources
   oc_obj:
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
index 318d34b5d..ea9b1fe1f 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
@@ -33,6 +33,7 @@
     params:
       IMAGE_NAME: "{{ glusterfs_heketi_image }}"
       IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
+      HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
index 3a9619d9d..26343b909 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -103,6 +103,7 @@
     params:
       IMAGE_NAME: "{{ glusterfs_heketi_image }}"
       IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
+      HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
@@ -124,7 +125,7 @@
 
 - name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
+    glusterfs_heketi_client: "oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
 
 - name: Verify heketi service
   command: "{{ glusterfs_heketi_client }} cluster list"
diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml
index c9bfdd1cd..d2d8c6c10 100644
--- a/roles/openshift_storage_glusterfs/tasks/main.yml
+++ b/roles/openshift_storage_glusterfs/tasks/main.yml
@@ -11,7 +11,7 @@
 
 - include: glusterfs_registry.yml
   when:
-  - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap"
+  - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.storage.glusterfs.swap"
 
 - name: Delete temp directory
   file:
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
index 9b8fae310..5ea801e60 100644
--- a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
@@ -5,6 +5,7 @@ metadata:
   name: glusterfs-{{ glusterfs_name }}
 provisioner: kubernetes.io/glusterfs
 parameters:
-  resturl: "http://{{ glusterfs_heketi_url }}:8081"
+  resturl: "http://{% if glusterfs_heketi_is_native %}{{ glusterfs_heketi_route }}{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %}"
+  restuser: "admin"
   secretNamespace: "{{ glusterfs_namespace }}"
-  secretName: "heketi-{{ glusterfs_name }}-user-secret"
+  secretName: "heketi-{{ glusterfs_name }}-secret"