From 47f350bc3aa85a8bd406d95faf084df2abf74ae9 Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Sun, 18 Mar 2018 22:59:31 +0100 Subject: Second revision: includes hostpath mounts, gluster block storage, kaas apps, etc. --- setup/configs/volumes.yml | 16 +++++- setup/projects/adei/files/adei_init/mysql/adei.sql | 2 + .../projects/adei/files/adei_init/mysql/initdb.sh | 4 +- setup/projects/adei/templates/01-secret.yml.j2 | 10 ++++ setup/projects/adei/templates/60-adei.yml.j2 | 23 ++++++++ setup/projects/adei/vars/galera.yml | 66 ++++++++++++++++++++++ setup/projects/adei/vars/globals.yml | 1 + setup/projects/adei/vars/pods.yml | 29 +++++++--- setup/projects/adei/vars/volumes.yml | 31 +++++----- .../projects/kaas/templates/40-kaas-manager.yml.j2 | 2 +- 10 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 setup/projects/adei/vars/galera.yml (limited to 'setup') diff --git a/setup/configs/volumes.yml b/setup/configs/volumes.yml index 14aadfa..020c7d2 100644 --- a/setup/configs/volumes.yml +++ b/setup/configs/volumes.yml @@ -1,5 +1,6 @@ --- ands_paths: + hostraid: /mnt/hostraid provision: /mnt/provision openshift: /mnt/openshift temporary: /mnt/temporary @@ -24,12 +25,14 @@ ands_storage_domains: - servers: "ands_storage_servers" clients: [ "masters", "new_masters" ] volumes: -# provision: { type: "cfg", mount: "{{ ands_paths.provision }}" } openshift: { type: "cfg", mount: "{{ ands_paths.openshift }}", nfs_clients: "{{ ands_nfs_clients }}" } - databases: { type: "db", mount: "{{ ands_paths.databases }}" } + databases: { type: "db", mount: "{{ ands_paths.databases }}", access: "ReadOnlyMany" } temporary: { type: "tmp", mount: "{{ ands_paths.temporary }}", nfs_clients: "{{ ands_nfs_clients }}" } datastore: { type: "data", mount: "{{ ands_paths.datastore }}", nfs_clients: "{{ ands_nfs_clients }}" } katrin_data: { type: "data", mount: "{{ ands_paths.katrin_data }}", nfs_clients: "{{ ands_nfs_clients }}" } + - servers: "ands_storage_servers" + volumes: + block: { type: "db", transport: "{{ ands_rdma_support | ternary('rdma', 'tcp') }}" } # - servers: "ands_storage_servers" # clients: [ "nodes", "new_nodes" ] @@ -39,6 +42,10 @@ ands_storage_domains: # - ovirt: # - pdv: +ands_local_storage_domains: + - servers: [ "ands_storage_servers" ] + volumes: + hostraid: { type: "host", path: "/mnt/ands/hostmount", mount: "{{ ands_paths.hostraid }}" } # Per project list (to distribute in multiple namespaces later) # If not started with '/' will be prepended with project name @@ -48,7 +55,12 @@ ands_openshift_volumes: data: { volume: "datastore", path: "", write: true } db: { volume: "databases", path: "", write: true } tmp: { volume: "temporary", path: "", write: true } + host: { volume: "hostraid", path: "", write: true } # Global list, we only take things from the volume of project #ands_openshift_files: # - { osv: "log", path: "apache2-kaas", state: "directory", mode: "0777" } + + +#ands_block_volumes: +# adei-mysql: { volume: "block", capacity: "2Ti", ha: 2, project: "kaas" } diff --git a/setup/projects/adei/files/adei_init/mysql/adei.sql b/setup/projects/adei/files/adei_init/mysql/adei.sql index a17fcfe..5bd7e8f 100644 --- a/setup/projects/adei/files/adei_init/mysql/adei.sql +++ b/setup/projects/adei/files/adei_init/mysql/adei.sql @@ -1 +1,3 @@ GRANT ALL ON `adei_%`.* TO 'adei'@'%'; +UPDATE mysql.user SET Super_Priv='Y' WHERE user='adei' AND host='%'; +FLUSH PRIVILEGES; diff --git a/setup/projects/adei/files/adei_init/mysql/initdb.sh b/setup/projects/adei/files/adei_init/mysql/initdb.sh index f877520..2790c2d 100644 --- a/setup/projects/adei/files/adei_init/mysql/initdb.sh +++ b/setup/projects/adei/files/adei_init/mysql/initdb.sh @@ -11,6 +11,6 @@ done - cat adei.sql | awk "{ gsub(/@PWD@/, \"$PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME - #cat pma.sql | awk "{ gsub(/@PWD@/, \"$PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME + cat adei.sql | awk "{ gsub(/@PWD@/, \"$MYSQL_PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME + #cat pma.sql | awk "{ gsub(/@PWD@/, \"$MYSQL_PMA_PASSWORD\"); print }" | MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -u root -h $HOSTNAME ) diff --git a/setup/projects/adei/templates/01-secret.yml.j2 b/setup/projects/adei/templates/01-secret.yml.j2 index f310ec9..44d5914 100644 --- a/setup/projects/adei/templates/01-secret.yml.j2 +++ b/setup/projects/adei/templates/01-secret.yml.j2 @@ -12,10 +12,14 @@ objects: metadata: annotations: template.openshift.io/expose-adei_password: '{.data[''adei-password'']}' + template.openshift.io/expose-root_password: '{.data[''root-password'']}' + template.openshift.io/expose-service_password: '{.data[''service-password'']}' template.openshift.io/expose-pma_password: '{.data[''pma-password'']}' name: adei stringData: adei-password: "{{ kaas_project_config.adei_password }}" + root-password: "{{ kaas_project_config.adei_password }}" + service-password: "${SERVICE_PASSWORD}" pma-password: "${PMA_PASSWORD}" parameters: - description: Password for the PMA connection user. @@ -24,3 +28,9 @@ parameters: generate: expression name: PMA_PASSWORD required: true +- description: Password for the service users + displayName: Service Connection Password + from: '[a-zA-Z0-9]{16}' + generate: expression + name: SERVICE_PASSWORD + required: true diff --git a/setup/projects/adei/templates/60-adei.yml.j2 b/setup/projects/adei/templates/60-adei.yml.j2 index 22f4bb0..7eafd33 100644 --- a/setup/projects/adei/templates/60-adei.yml.j2 +++ b/setup/projects/adei/templates/60-adei.yml.j2 @@ -159,6 +159,29 @@ objects: {% endif %} env: {{ cfg.env | to_json }} volumeMounts: {{ cfg.mounts | to_json }} +{% if cfg.resources is defined %} + resources: +{% if cfg.resources.request is defined %} +{% set res = cfg.resources.request %} + requests: +{% if res.cpu %} + cpu: {{ res.cpu }} +{% endif %} +{% if res.cpu %} + memory: {{ res.mem }} +{% endif %} +{% endif %} +{% if cfg.resources.limit is defined %} +{% set res = cfg.resources.limit %} + limits: +{% if res.cpu %} + cpu: {{ res.cpu }} +{% endif %} +{% if res.cpu %} + memory: {{ res.mem }} +{% endif %} +{% endif %} +{% endif %} {% if (cfg.node is defined) %} livenessProbe: timeoutSeconds: 1 diff --git a/setup/projects/adei/vars/galera.yml b/setup/projects/adei/vars/galera.yml new file mode 100644 index 0000000..ea64daa --- /dev/null +++ b/setup/projects/adei/vars/galera.yml @@ -0,0 +1,66 @@ +galera_app: + name: galera + provision: true + instantiate: false + pods: + galera: + kind: StatefulSet + service: { ports: [ 3306 ] } + sched: { replicas: 3, strategy: "Recreate", restrict: { fat_storage: "1" } } + update: { strategy: RollingUpdate, min_ready: 30 } + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - { key: "hostid", operator: "In", values: [ "1", "2", "3" ] } + groups: [ "adei_db" ] + images: + - image: "chsa/mysql-galera:5.7" + command: [ "run-mysqld-galera" ] + ports: [ 3306, 4444, 4567, 4568 ] + env: + - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } + - { name: "MYSQL_USER", value: "adei" } + - { name: "MYSQL_USER_PRIV_SUPER", value: "1" } + - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } + - { name: "MYSQL_DATABASE", value: "adei" } + - { name: "MYSQL_EXTRADB", value: "adei_%" } + - { name: "MYSQL_GALERA_USER", value: "xtrabackup_sst" } + - { name: "MYSQL_GALERA_PASSWORD", value: "secret@adei/service-password" } + mappings: + - { name: "adei_init", mount: "/var/lib/init" } + - { name: "adei_host", path: "galera", mount: "/var/lib/mysql/data" } + resources: { request: { cpu: 2000m, mem: 4Gi }, limit: { cpu: 6000m, mem: 32Gi } } +# probes: +# - { type: "liveness", port: 3306 } +# - { type: "readiness", command: [ /bin/sh, -i, -c, MYSQL_PWD="$MYSQL_PASSWORD" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE, -e 'SELECT 1' ], delay: "15", timeout: "5" } + + + grecovery: + sched: { replicas: 0, strategy: "Recreate", restrict: { fat_storage: "1" } } + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - { key: "hostid", operator: "In", values: [ "1", "2", "3" ] } + groups: [ "adei_db" ] + images: + - image: "chsa/mysql-galera:5.7" + command: [ "run-mysqld-manager" ] + ports: [ 3306, 4444, 4567, 4568 ] + env: + - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } + - { name: "MYSQL_USER", value: "adei" } + - { name: "MYSQL_USER_PRIV_SUPER", value: "1" } + - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } + - { name: "MYSQL_DATABASE", value: "adei" } + - { name: "MYSQL_EXTRADB", value: "adei_%" } + - { name: "MYSQL_GALERA_USER", value: "xtrabackup_sst" } + - { name: "MYSQL_GALERA_PASSWORD", value: "secret@adei/service-password" } + - { name: "POD_NAMESPACE", value: "fieldref@metadata.namespace" } + - { name: "MYSQL_GALERA_CLUSTER", value: "galera-ss" } + mappings: + - { name: "adei_init", mount: "/var/lib/init" } + - { name: "adei_host", path: "galera", mount: "/var/lib/mysql/data" } diff --git a/setup/projects/adei/vars/globals.yml b/setup/projects/adei/vars/globals.yml index 01fb495..86911aa 100644 --- a/setup/projects/adei/vars/globals.yml +++ b/setup/projects/adei/vars/globals.yml @@ -182,6 +182,7 @@ adei_frontends: cacher: name: "adei-${setup}-cacher" replicas: "${cache_replicas}" + resources: { request: { cpu: 1000m, mem: 1Gi } } cmd: [ "/openshift-entrypoint.sh", "/adei/src/scripts/system/cacher.sh" ] env: "{{ adei_pod_env | union(adei_cache_env) }}" vols: "{{ adei_pod_vols }}" diff --git a/setup/projects/adei/vars/pods.yml b/setup/projects/adei/vars/pods.yml index 3923c23..8857fcd 100644 --- a/setup/projects/adei/vars/pods.yml +++ b/setup/projects/adei/vars/pods.yml @@ -1,24 +1,27 @@ pods: mysql: service: { ports: [ 3306 ] } - sched: { replicas: 1, strategy: "Recreate", restrict: { fat_storage: "1" } } + sched: { replicas: 1, strategy: "Recreate", selector: { hostid: "3" } } groups: [ "adei_db" ] images: - - image: "centos/mysql-57-centos7" - env: + - image: "centos/mysql-57-centos7" + env: - { name: "MYSQL_USER", value: "adei" } - { name: "MYSQL_PASSWORD", value: "secret@adei/adei-password" } - - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/adei-password" } + - { name: "MYSQL_ROOT_PASSWORD", value: "secret@adei/root-password" } - { name: "MYSQL_DATABASE", value: "adei" } - - { name: "PMA_PASSWORD", value: "secret@adei/pma-password" } - mappings: + - { name: "MYSQL_PMA_PASSWORD", value: "secret@adei/pma-password" } + - { name: "MYSQL_MAX_CONNECTIONS", value: "500" } + mappings: - { name: "adei_init", mount: "/var/lib/init" } - - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" } - probes: + - { name: "adei_host", path: "mysql", mount: "/var/lib/mysql/data" } +# - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" } + resources: { request: { cpu: 2000m, mem: 4Gi }, limit: { cpu: 6000m, mem: 32Gi } } + probes: - { port: 3306 } # - { type: "liveness", port: 3306 } # - { type: "readiness", command: [/bin/sh, -i, -c, MYSQL_PWD="$MYSQL_PASSWORD" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE, -e 'SELECT 1'] } - hooks: + hooks: - { type: "postStart", command: [ "/bin/bash", "/var/lib/init/mysql/initdb.sh" ] } phpmyadmin: @@ -35,6 +38,14 @@ pods: probes: - { port: 8080, path: '/' } + + +apps: + - "galera_app" + + + + #oc: # - template: "[0-3]*" # - template: "[4-6]*" diff --git a/setup/projects/adei/vars/volumes.yml b/setup/projects/adei/vars/volumes.yml index 768e27f..82f2e18 100644 --- a/setup/projects/adei/vars/volumes.yml +++ b/setup/projects/adei/vars/volumes.yml @@ -3,20 +3,23 @@ gids: adei_db: { id: 6002 } volumes: + adei_host: { volume: "hostraid", path: "/adei", write: true } # mysql adei_init: { volume: "openshift", path: "/adei/init"} # mysql - adei_etc: { volume: "openshift", path: "/adei/etc"} # mysql (maybe) - adei_src: { volume: "openshift", path: "/adei/src", write: true } # prod & debug (init creates setup links) - adei_cfg: { volume: "openshift", path: "/adei/cfg", write: true } # per-setup configs (ADEI/wiki modifies setup) - adei_sys: { volume: "openshift", path: "/adei/sys" } # per-setup cron-jon overrides - adei_tmp: { volume: "temporary", path: "/adei/tmp", write: true } # per-setup temporary files - adei_log: { volume: "temporary", path: "/adei/log", write: true } # per-replica (should be fine) temporary files - adei_db: { volume: "databases", path: "/adei", write: true } # mysql + adei_etc: { volume: "openshift", path: "/adei/etc"} # mysql (maybe) + adei_src: { volume: "openshift", path: "/adei/src", write: true } # prod & debug (init creates setup links) + adei_cfg: { volume: "openshift", path: "/adei/cfg", write: true } # per-setup configs (ADEI/wiki modifies setup) + adei_sys: { volume: "openshift", path: "/adei/sys" } # per-setup cron-jon overrides + adei_tmp: { volume: "temporary", path: "/adei/tmp", write: true } # per-setup temporary files + adei_log: { volume: "temporary", path: "/adei/log", write: true } # per-replica (should be fine) temporary files +# adei_db: { volume: "databases", path: "/adei", write: true } # mysql files: - - { osv: "adei_cfg", path: "/", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_src", path: "/", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_src", path: "/prod", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_src", path: "/dbg", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_log", path: "/", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_tmp", path: "/", state: "directory", group: "adei", mode: "02775" } - - { osv: "adei_db", path: "mysql", state: "directory", group: "adei_db", mode: "02775" } + - { osv: "adei_cfg", path: "/", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_src", path: "/", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_src", path: "/prod", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_src", path: "/dbg", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_log", path: "/", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_tmp", path: "/", state: "directory", group: "adei", mode: "02775" } + - { osv: "adei_host",path: "mysql", state: "directory", group: "adei_db", mode: "02775" } + - { osv: "adei_host",path: "galera", state: "directory", group: "adei_db", mode: "02775" } +# - { osv: "adei_db", path: "mysql", state: "directory", group: "adei_db", mode: "02775" } diff --git a/setup/projects/kaas/templates/40-kaas-manager.yml.j2 b/setup/projects/kaas/templates/40-kaas-manager.yml.j2 index b9cba4e..0e0f45e 100644 --- a/setup/projects/kaas/templates/40-kaas-manager.yml.j2 +++ b/setup/projects/kaas/templates/40-kaas-manager.yml.j2 @@ -13,7 +13,7 @@ objects: metadata: name: kaas-manager spec: - replicas: 1 + replicas: 0 revisionHistoryLimit: {{ kaas_pod_history_limit }} strategy: type: Rolling -- cgit v1.2.3