Second revision: includes hostpath mounts, gluster block storage, kaas apps, etc.

10 files changed, 156 insertions, 28 deletions

diff --git a/setup/configs/volumes.yml b/setup/configs/volumes.yml
index 14aadfa..020c7d2 100644
--- a/setup/configs/volumes.yml
+++ b/setup/configs/volumes.yml
@@ -1,5 +1,6 @@
 ---
 ands_paths:
+  hostraid: /mnt/hostraid
   provision: /mnt/provision
   openshift: /mnt/openshift
   temporary: /mnt/temporary
@@ -24,12 +25,14 @@ ands_storage_domains:
   - servers: "ands_storage_servers"
     clients: [ "masters", "new_masters" ]
     volumes:
-#      provision: { type: "cfg",  mount: "{{ ands_paths.provision }}" }
       openshift: { type: "cfg",  mount: "{{ ands_paths.openshift }}", nfs_clients: "{{ ands_nfs_clients }}" }
-      databases: { type: "db",   mount: "{{ ands_paths.databases }}" }
+      databases: { type: "db",   mount: "{{ ands_paths.databases }}", access: "ReadOnlyMany" }
       temporary: { type: "tmp",  mount: "{{ ands_paths.temporary }}", nfs_clients: "{{ ands_nfs_clients }}" }
       datastore: { type: "data", mount: "{{ ands_paths.datastore }}", nfs_clients: "{{ ands_nfs_clients }}" }
       katrin_data: { type: "data", mount: "{{ ands_paths.katrin_data }}", nfs_clients: "{{ ands_nfs_clients }}" }
+  - servers: "ands_storage_servers"
+    volumes:
+      block:    { type: "db", transport: "{{ ands_rdma_support | ternary('rdma', 'tcp') }}" }
 
 #  - servers: "ands_storage_servers"
 #    clients: [ "nodes", "new_nodes" ]
@@ -39,6 +42,10 @@ ands_storage_domains:
 #  - ovirt:
 #  - pdv:
 
+ands_local_storage_domains:
+  - servers: [ "ands_storage_servers" ]
+    volumes:
+      hostraid: { type: "host", path: "/mnt/ands/hostmount", mount: "{{ ands_paths.hostraid }}" }
 
 # Per project list (to distribute in multiple namespaces later)
 # If not started with '/' will be prepended with project name
@@ -48,7 +55,12 @@ ands_openshift_volumes:
   data: { volume: "datastore", path: "", write: true }
   db:   { volume: "databases", path: "", write: true }
   tmp:  { volume: "temporary", path: "", write: true }
+  host: { volume: "hostraid",  path: "", write: true }
 
 # Global list, we only take things from the volume of project
 #ands_openshift_files:
 #  - { osv: "log", path: "apache2-kaas", state: "directory", mode: "0777" }
+
+
+#ands_block_volumes:
+#  adei-mysql: { volume: "block", capacity: "2Ti", ha: 2, project: "kaas" }
diff --git a/setup/projects/adei/files/adei_init/mysql/adei.sql b/setup/projects/adei/files/adei_init/mysql/adei.sql
index a17fcfe..5bd7e8f 100644
--- a/setup/projects/adei/files/adei_init/mysql/adei.sql
+++ b/setup/projects/adei/files/adei_init/mysql/adei.sql
@@ -1 +1,3 @@
 GRANT ALL ON `adei_%`.* TO 'adei'@'%';
+UPDATE mysql.user SET Super_Priv='Y' WHERE user='adei' AND host='%';
+FLUSH PRIVILEGES;
diff --git a/setup/projects/adei/files/adei_init/mysql/initdb.sh b/setup/projects/adei/files/adei_init/mysql/initdb.sh
index f877520..2790c2d 100644
--- a/setup/projects/adei/files/adei_init/mysql/initdb.sh
+++ b/setup/projects/adei/files/adei_init/mysql/initdb.sh
@@ -11,6 +11,6 @@
     done
 
 
-    cat adei.sql | awk "{ gsub(/@PWD@/, \"$PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME
-    #cat pma.sql | awk "{ gsub(/@PWD@/, \"$PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME
+    cat adei.sql | awk "{ gsub(/@PWD@/, \"$MYSQL_PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME
+    #cat pma.sql | awk "{ gsub(/@PWD@/, \"$MYSQL_PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME
 )
diff --git a/setup/projects/adei/templates/01-secret.yml.j2 b/setup/projects/adei/templates/01-secret.yml.j2
index f310ec9..44d5914 100644
--- a/setup/projects/adei/templates/01-secret.yml.j2
+++ b/setup/projects/adei/templates/01-secret.yml.j2
@@ -12,10 +12,14 @@ objects:
   metadata:
     annotations:
       template.openshift.io/expose-adei_password: '{.data[''adei-password'']}'
+      template.openshift.io/expose-root_password: '{.data[''root-password'']}'
+      template.openshift.io/expose-service_password: '{.data[''service-password'']}'
       template.openshift.io/expose-pma_password: '{.data[''pma-password'']}'
     name: adei
   stringData:
     adei-password: "{{ kaas_project_config.adei_password }}"
+    root-password: "{{ kaas_project_config.adei_password }}"
+    service-password: "${SERVICE_PASSWORD}"
     pma-password: "${PMA_PASSWORD}"
 parameters:
 - description: Password for the PMA connection user.
@@ -24,3 +28,9 @@ parameters:
   generate: expression
   name: PMA_PASSWORD
   required: true
+- description: Password for the service users
+  displayName: Service Connection Password
+  from: '[a-zA-Z0-9]{16}'
+  generate: expression
+  name: SERVICE_PASSWORD
+  required: true
diff --git a/setup/projects/adei/templates/60-adei.yml.j2 b/setup/projects/adei/templates/60-adei.yml.j2
index 22f4bb0..7eafd33 100644
--- a/setup/projects/adei/templates/60-adei.yml.j2
+++ b/setup/projects/adei/templates/60-adei.yml.j2
@@ -159,6 +159,29 @@ objects:
 {%     endif %}
                     env: {{ cfg.env | to_json }}
                     volumeMounts: {{ cfg.mounts | to_json }}
+{%     if cfg.resources is defined %}
+                    resources:
+{%       if cfg.resources.request is defined %}
+{%         set res = cfg.resources.request %}
+                      requests:
+{%         if res.cpu %}
+                        cpu: {{ res.cpu }}
+{%         endif %}
+{%         if res.cpu %}
+                        memory: {{ res.mem }}
+{%         endif %}
+{%       endif %}
+{%       if cfg.resources.limit is defined %}
+{%         set res = cfg.resources.limit %}
+                      limits:
+{%         if res.cpu %}
+                        cpu: {{ res.cpu }}
+{%         endif %}
+{%         if res.cpu %}
+                        memory: {{ res.mem }}
+{%         endif %}
+{%       endif %}
+{%     endif %}
 {%     if (cfg.node is defined)  %}
                     livenessProbe:
                       timeoutSeconds: 1
diff --git a/setup/projects/adei/vars/galera.yml b/setup/projects/adei/vars/galera.yml
new file mode 100644
index 0000000..ea64daa
--- /dev/null
+++ b/setup/projects/adei/vars/galera.yml
@@ -0,0 +1,66 @@
+galera_app:
+  name: galera
+  provision: true
+  instantiate: false
+  pods:
+    galera:
+      kind: StatefulSet
+      service: { ports: [ 3306 ] }
+      sched: { replicas: 3, strategy: "Recreate", restrict: { fat_storage: "1" } }
+      update: { strategy: RollingUpdate, min_ready: 30 }
+      affinity: 
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution: 
+            nodeSelectorTerms:
+            - matchExpressions:
+              - { key: "hostid", operator: "In", values: [ "1", "2", "3" ] }
+      groups: [ "adei_db" ]
+      images:
+        - image: "chsa/mysql-galera:5.7"
+          command: [ "run-mysqld-galera" ]
+          ports: [ 3306, 4444, 4567, 4568 ]
+          env:
+            - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } 
+            - { name: "MYSQL_USER", value: "adei" }
+            - { name: "MYSQL_USER_PRIV_SUPER", value: "1" }
+            - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } 
+            - { name: "MYSQL_DATABASE", value: "adei" }
+            - { name: "MYSQL_EXTRADB", value: "adei_%" }
+            - { name: "MYSQL_GALERA_USER", value: "xtrabackup_sst" }
+            - { name: "MYSQL_GALERA_PASSWORD", value: "secret@adei/service-password" }
+          mappings: 
+            - { name: "adei_init", mount: "/var/lib/init" }
+            - { name: "adei_host", path: "galera", mount: "/var/lib/mysql/data" }
+          resources: {  request: { cpu: 2000m, mem: 4Gi }, limit: { cpu: 6000m, mem: 32Gi } }
+#          probes:
+#            - { type: "liveness", port: 3306 }
+#            - { type: "readiness", command: [ /bin/sh, -i, -c, MYSQL_PWD="$MYSQL_PASSWORD" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE, -e 'SELECT 1' ], delay: "15", timeout: "5" }
+
+
+    grecovery:
+      sched: { replicas: 0, strategy: "Recreate", restrict: { fat_storage: "1" } }
+      affinity: 
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution: 
+            nodeSelectorTerms:
+            - matchExpressions:
+              - { key: "hostid", operator: "In", values: [ "1", "2", "3" ] }
+      groups: [ "adei_db" ]
+      images:
+        - image: "chsa/mysql-galera:5.7"
+          command: [ "run-mysqld-manager" ]
+          ports: [ 3306, 4444, 4567, 4568 ]
+          env:
+            - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } 
+            - { name: "MYSQL_USER", value: "adei" }
+            - { name: "MYSQL_USER_PRIV_SUPER", value: "1" }
+            - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } 
+            - { name: "MYSQL_DATABASE", value: "adei" }
+            - { name: "MYSQL_EXTRADB", value: "adei_%" }
+            - { name: "MYSQL_GALERA_USER", value: "xtrabackup_sst" }
+            - { name: "MYSQL_GALERA_PASSWORD", value: "secret@adei/service-password" }
+            - { name: "POD_NAMESPACE", value: "fieldref@metadata.namespace" }
+            - { name: "MYSQL_GALERA_CLUSTER", value: "galera-ss" }
+          mappings: 
+            - { name: "adei_init", mount: "/var/lib/init" }
+            - { name: "adei_host", path: "galera", mount: "/var/lib/mysql/data" }
diff --git a/setup/projects/adei/vars/globals.yml b/setup/projects/adei/vars/globals.yml
index 01fb495..86911aa 100644
--- a/setup/projects/adei/vars/globals.yml
+++ b/setup/projects/adei/vars/globals.yml
@@ -182,6 +182,7 @@ adei_frontends:
     cacher:
       name: "adei-${setup}-cacher"
       replicas: "${cache_replicas}"
+      resources: {  request: { cpu: 1000m, mem: 1Gi } }
       cmd: [ "/openshift-entrypoint.sh", "/adei/src/scripts/system/cacher.sh" ]
       env: "{{ adei_pod_env | union(adei_cache_env) }}"
       vols: "{{ adei_pod_vols }}"
diff --git a/setup/projects/adei/vars/pods.yml b/setup/projects/adei/vars/pods.yml
index 3923c23..8857fcd 100644
--- a/setup/projects/adei/vars/pods.yml
+++ b/setup/projects/adei/vars/pods.yml
@@ -1,24 +1,27 @@
 pods:
   mysql:
     service: { ports: [ 3306 ] }
-    sched: { replicas: 1, strategy: "Recreate", restrict: { fat_storage: "1" } }
+    sched: { replicas: 1, strategy: "Recreate", selector: { hostid: "3" } }
     groups: [ "adei_db" ]
     images:
-      - image: "centos/mysql-57-centos7"
-        env:
+    - image: "centos/mysql-57-centos7"
+      env:
             - { name: "MYSQL_USER", value: "adei" }
             - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } 
-            - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } 
+            - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/root-password" } 
             - { name: "MYSQL_DATABASE", value: "adei" }
-            - { name: "PMA_PASSWORD", value: "secret@adei/pma-password" } 
-        mappings: 
+            - { name: "MYSQL_PMA_PASSWORD", value: "secret@adei/pma-password" } 
+            - { name: "MYSQL_MAX_CONNECTIONS", value: "500" }
+      mappings: 
             - { name: "adei_init", mount: "/var/lib/init" }
-            - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" }
-        probes:
+            - { name: "adei_host", path: "mysql", mount: "/var/lib/mysql/data" }
+#            - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" }
+      resources: {  request: { cpu: 2000m, mem: 4Gi }, limit: { cpu: 6000m, mem: 32Gi } }
+      probes:
             - { port: 3306 }
 #            - { type: "liveness", port: 3306 }
 #            - { type: "readiness", command: [/bin/sh, -i, -c, MYSQL_PWD="$MYSQL_PASSWORD" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE, -e 'SELECT 1'] }
-        hooks:
+      hooks:
             - { type: "postStart", command: [ "/bin/bash", "/var/lib/init/mysql/initdb.sh" ] }
 
   phpmyadmin:
@@ -35,6 +38,14 @@ pods:
         probes:
             - { port: 8080, path: '/' }
 
+
+
+apps:
+ - "galera_app"
+ 
+
+
+
 #oc:
 #  - template: "[0-3]*"
 #  - template: "[4-6]*"
diff --git a/setup/projects/adei/vars/volumes.yml b/setup/projects/adei/vars/volumes.yml
index 768e27f..82f2e18 100644
--- a/setup/projects/adei/vars/volumes.yml
+++ b/setup/projects/adei/vars/volumes.yml
@@ -3,20 +3,23 @@ gids:
   adei_db: { id: 6002 }
 
 volumes:
+  adei_host: { volume: "hostraid",  path: "/adei",       write: true }                  # mysql
   adei_init: { volume: "openshift", path: "/adei/init"}                                 # mysql
-  adei_etc: { volume: "openshift", path: "/adei/etc"}                                   # mysql (maybe)
-  adei_src: { volume: "openshift", path: "/adei/src",   write: true }                   # prod & debug (init creates setup links)
-  adei_cfg: { volume: "openshift", path: "/adei/cfg",   write: true }                   # per-setup configs (ADEI/wiki modifies setup)
-  adei_sys: { volume: "openshift", path: "/adei/sys" }                                  # per-setup cron-jon overrides
-  adei_tmp: { volume: "temporary", path: "/adei/tmp",   write: true }                   # per-setup temporary files
-  adei_log: { volume: "temporary", path: "/adei/log",   write: true }                   # per-replica (should be fine) temporary files
-  adei_db:  { volume: "databases", path: "/adei",       write: true }                   # mysql
+  adei_etc:  { volume: "openshift", path: "/adei/etc"}                                  # mysql (maybe)
+  adei_src:  { volume: "openshift", path: "/adei/src",   write: true }                  # prod & debug (init creates setup links)
+  adei_cfg:  { volume: "openshift", path: "/adei/cfg",   write: true }                  # per-setup configs (ADEI/wiki modifies setup)
+  adei_sys:  { volume: "openshift", path: "/adei/sys" }                                 # per-setup cron-jon overrides
+  adei_tmp:  { volume: "temporary", path: "/adei/tmp",   write: true }                  # per-setup temporary files
+  adei_log:  { volume: "temporary", path: "/adei/log",   write: true }                  # per-replica (should be fine) temporary files
+#  adei_db:   { volume: "databases", path: "/adei",       write: true }                  # mysql
 
 files:
-  - { osv: "adei_cfg", path: "/",     state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_src", path: "/",     state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_src", path: "/prod", state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_src", path: "/dbg",  state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_log", path: "/",     state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_tmp", path: "/",     state: "directory", group: "adei", mode: "02775" }
-  - { osv: "adei_db", path: "mysql",  state: "directory", group: "adei_db", mode: "02775" }
+  - { osv: "adei_cfg", path: "/",       state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_src", path: "/",       state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_src", path: "/prod",   state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_src", path: "/dbg",    state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_log", path: "/",       state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_tmp", path: "/",       state: "directory", group: "adei", mode: "02775" }
+  - { osv: "adei_host",path: "mysql",   state: "directory", group: "adei_db", mode: "02775" }
+  - { osv: "adei_host",path: "galera",  state: "directory", group: "adei_db", mode: "02775" }
+#  - { osv: "adei_db",  path: "mysql",   state: "directory", group: "adei_db", mode: "02775" }
diff --git a/setup/projects/kaas/templates/40-kaas-manager.yml.j2 b/setup/projects/kaas/templates/40-kaas-manager.yml.j2
index b9cba4e..0e0f45e 100644
--- a/setup/projects/kaas/templates/40-kaas-manager.yml.j2
+++ b/setup/projects/kaas/templates/40-kaas-manager.yml.j2
@@ -13,7 +13,7 @@ objects:
     metadata:
       name: kaas-manager
     spec:
-      replicas: 1
+      replicas: 0
       revisionHistoryLimit: {{ kaas_pod_history_limit }}
       strategy:
         type: Rolling