Merge pull request #6541 from smarterclayton/move_gcp_in

Allow openshift-ansible image to deploy to GCP

20 files changed, 394 insertions, 5 deletions

diff --git a/roles/openshift_gcp/files/bootstrap-script.sh b/roles/openshift_gcp/files/bootstrap-script.sh
new file mode 100644
index 000000000..0c3f1999b
--- /dev/null
+++ b/roles/openshift_gcp/files/bootstrap-script.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# This script is a startup script for bootstrapping a GCP node
+# from a config stored in the project metadata. It loops until
+# it finds the script and then starts the origin-node service.
+# TODO: generalize
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap" -H "Metadata-Flavor: Google" )" != "true" ]]; then
+  echo "info: Bootstrap is not enabled for this instance, skipping" 1>&2
+  exit 0
+fi
+
+if ! id=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-id" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get cluster-id for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! node_group=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/node-group" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get node-group for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+  while true; do
+    if config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/project/attributes/${id}-bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+      break
+    fi
+    echo "info: waiting for ${id}-bootstrap-config to become available in cluster metadata ..." 1>&2
+    sleep 5
+  done
+fi
+
+echo "Got bootstrap config from metadata"
+mkdir -p /etc/origin/node
+echo -n "${config}" > /etc/origin/node/bootstrap.kubeconfig
+echo "BOOTSTRAP_CONFIG_NAME=node-config-${node_group}" >> /etc/sysconfig/origin-node
+systemctl enable origin-node
+systemctl start origin-node
diff --git a/roles/openshift_gcp/files/openshift-bootstrap-update.service b/roles/openshift_gcp/files/openshift-bootstrap-update.service
new file mode 100644
index 000000000..c65b1b34e
--- /dev/null
+++ b/roles/openshift_gcp/files/openshift-bootstrap-update.service
@@ -0,0 +1,7 @@
+[Unit]
+Description=Update the OpenShift node bootstrap configuration
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/openshift-bootstrap-update
+User=root
diff --git a/roles/openshift_gcp/files/openshift-bootstrap-update.timer b/roles/openshift_gcp/files/openshift-bootstrap-update.timer
new file mode 100644
index 000000000..1a517b33e
--- /dev/null
+++ b/roles/openshift_gcp/files/openshift-bootstrap-update.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Update the OpenShift node bootstrap credentials hourly
+
+[Timer]
+OnBootSec=30s
+OnCalendar=hourly
+Persistent=true     
+ 
+[Install]
+WantedBy=timers.target
\ No newline at end of file
diff --git a/roles/openshift_gcp/files/partition.conf b/roles/openshift_gcp/files/partition.conf
new file mode 100644
index 000000000..76e65ab9c
--- /dev/null
+++ b/roles/openshift_gcp/files/partition.conf
@@ -0,0 +1,3 @@
+[Service]
+ExecStartPost=-/usr/bin/growpart /dev/sda 1
+ExecStartPost=-/sbin/xfs_growfs /
\ No newline at end of file
diff --git a/roles/openshift_gcp/meta/main.yml b/roles/openshift_gcp/meta/main.yml
new file mode 100644
index 000000000..5e428f8de
--- /dev/null
+++ b/roles/openshift_gcp/meta/main.yml
@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  author: Clayton Coleman
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- role: lib_utils
+- role: lib_openshift
diff --git a/roles/openshift_gcp/tasks/add_custom_repositories.yml b/roles/openshift_gcp/tasks/add_custom_repositories.yml
new file mode 100644
index 000000000..04718f78e
--- /dev/null
+++ b/roles/openshift_gcp/tasks/add_custom_repositories.yml
@@ -0,0 +1,20 @@
+---
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientcert }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_cert
+  when: item.1.sslclientcert | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientkey }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_key
+  when: item.1.sslclientkey | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+
+- name: Create any custom repos that are defined
+  template:
+    src: yum_repo.j2
+    dest: /etc/yum.repos.d/provision_custom_repositories.repo
+  when: provision_custom_repositories | length > 0
+  notify: refresh cache
diff --git a/roles/openshift_gcp/tasks/configure_gcp_base_image.yml b/roles/openshift_gcp/tasks/configure_gcp_base_image.yml
new file mode 100644
index 000000000..2c6e2790a
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_gcp_base_image.yml
@@ -0,0 +1,10 @@
+# GCE instances are starting with xfs AND barrier=1, which is only for extfs.
+---
+- name: Remove barrier=1 from XFS fstab entries
+  command: sed -i -e 's/xfs\(.*\)barrier=1/xfs\1/g; s/, / /g' /etc/fstab
+
+- name: Ensure the root filesystem has XFS group quota turned on
+  command: sed -i -e 's/linux16 \(.*\)$/linux16 \1 rootflags=gquota/g' /boot/grub2/grub.cfg
+
+- name: Ensure the root partition grows on startup
+  copy: src=partition.conf dest=/etc/systemd/system/google-instance-setup.service.d/
diff --git a/roles/openshift_gcp/tasks/configure_master_bootstrap.yml b/roles/openshift_gcp/tasks/configure_master_bootstrap.yml
new file mode 100644
index 000000000..591cb593c
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_master_bootstrap.yml
@@ -0,0 +1,36 @@
+#
+# These tasks configure the instance to periodically update the project metadata with the
+# latest bootstrap kubeconfig from the project metadata. This keeps the project metadata
+# in sync with the cluster's configuration. We then invoke a CSR approve on any nodes that
+# are waiting to join the cluster.
+#
+---
+- name: Copy unit service
+  copy:
+    src: openshift-bootstrap-update.timer
+    dest: /etc/systemd/system/openshift-bootstrap-update.timer
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Copy unit timer
+  copy:
+    src: openshift-bootstrap-update.service
+    dest: /etc/systemd/system/openshift-bootstrap-update.service
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Create bootstrap update script
+  template: src=openshift-bootstrap-update.j2 dest=/usr/bin/openshift-bootstrap-update mode=u+rx
+
+- name: Start bootstrap update timer
+  systemd:
+    name: "openshift-bootstrap-update.timer"
+    state: started
+
+- name: Bootstrap all nodes that were identified with bootstrap metadata
+  run_once: true
+  oc_adm_csr:
+    nodes: "{{ groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list }}"
+    timeout: 60
diff --git a/roles/openshift_gcp/tasks/configure_master_healthcheck.yml b/roles/openshift_gcp/tasks/configure_master_healthcheck.yml
new file mode 100644
index 000000000..aa9655977
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_master_healthcheck.yml
@@ -0,0 +1,19 @@
+---
+- name: refresh yum cache
+  command: yum clean all
+  args:
+    warn: no
+  when: ansible_os_family == "RedHat"
+
+- name: install haproxy
+  package: name=haproxy state=present
+  register: result
+  until: '"failed" not in result'
+  retries: 10
+  delay: 10
+
+- name: configure haproxy
+  template: src=master_healthcheck.j2 dest=/etc/haproxy/haproxy.cfg
+
+- name: start and enable haproxy service
+  service: name=haproxy state=started enabled=yes
diff --git a/roles/openshift_gcp/tasks/dynamic_inventory.yml b/roles/openshift_gcp/tasks/dynamic_inventory.yml
new file mode 100644
index 000000000..1637da945
--- /dev/null
+++ b/roles/openshift_gcp/tasks/dynamic_inventory.yml
@@ -0,0 +1,5 @@
+---
+- name: Extract PEM from service account file
+  copy: content="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}" dest=/tmp/gce.pem mode=0600
+- name: Templatize environment script
+  template: src=inventory.j2.sh dest=/tmp/inventory.sh mode=u+rx
diff --git a/roles/openshift_gcp/tasks/frequent_log_rotation.yml b/roles/openshift_gcp/tasks/frequent_log_rotation.yml
new file mode 100644
index 000000000..0b4b27f84
--- /dev/null
+++ b/roles/openshift_gcp/tasks/frequent_log_rotation.yml
@@ -0,0 +1,18 @@
+---
+- name: Rotate logs daily
+  replace:
+    dest: /etc/logrotate.conf
+    regexp: '^weekly|monthly|yearly$'
+    replace: daily
+- name: Rotate at a smaller size of log
+  lineinfile:
+    dest: /etc/logrotate.conf
+    state: present
+    regexp: '^size'
+    line: size 10M
+- name: Limit total size of log files
+  lineinfile:
+    dest: /etc/logrotate.conf
+    state: present
+    regexp: '^maxsize'
+    line: maxsize 20M
diff --git a/roles/openshift_gcp/tasks/main.yaml b/roles/openshift_gcp/tasks/main.yml
index ad205ba33..fb147bc78 100644
--- a/roles/openshift_gcp/tasks/main.yaml
+++ b/roles/openshift_gcp/tasks/main.yml
@@ -17,7 +17,7 @@
 - name: Provision GCP DNS domain
   command: /tmp/openshift_gcp_provision_dns.sh
   args:
-    chdir: "{{ playbook_dir }}/files"
+    chdir: "{{ files_dir }}"
   register: dns_provision
   when:
   - state | default('present') == 'present'
@@ -33,7 +33,7 @@
 - name: Provision GCP resources
   command: /tmp/openshift_gcp_provision.sh
   args:
-    chdir: "{{ playbook_dir }}/files"
+    chdir: "{{ files_dir }}"
   when:
   - state | default('present') == 'present'
 
diff --git a/roles/openshift_gcp/tasks/node_cloud_config.yml b/roles/openshift_gcp/tasks/node_cloud_config.yml
new file mode 100644
index 000000000..4e982f497
--- /dev/null
+++ b/roles/openshift_gcp/tasks/node_cloud_config.yml
@@ -0,0 +1,12 @@
+---
+- name: ensure the /etc/origin folder exists
+  file: name=/etc/origin state=directory
+
+- name: configure gce cloud config options
+  ini_file: dest=/etc/origin/cloudprovider/gce.conf section=Global option={{ item.key }} value={{ item.value }} state=present create=yes
+  with_items:
+    - { key: 'project-id', value: '{{ openshift_gcp_project }}' }
+    - { key: 'network-name', value: '{{ openshift_gcp_network_name }}' }
+    - { key: 'node-tags', value: '{{ openshift_gcp_prefix }}ocp' }
+    - { key: 'node-instance-prefix', value: '{{ openshift_gcp_prefix }}' }
+    - { key: 'multizone', value: 'false' }
diff --git a/roles/openshift_gcp/tasks/publish_image.yml b/roles/openshift_gcp/tasks/publish_image.yml
new file mode 100644
index 000000000..db8a7ca69
--- /dev/null
+++ b/roles/openshift_gcp/tasks/publish_image.yml
@@ -0,0 +1,32 @@
+---
+- name: Require openshift_gcp_image
+  fail:
+    msg: "A source image name or family is required for image publishing.  Please ensure `openshift_gcp_image` is defined."
+  when: openshift_gcp_image is undefined
+
+- name: Require openshift_gcp_target_image
+  fail:
+    msg: "A target image name or family is required for image publishing.  Please ensure `openshift_gcp_target_image` is defined."
+  when: openshift_gcp_target_image is undefined
+
+- block:
+  - name: Retrieve images in the {{ openshift_gcp_target_image }} family
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images list
+        "--filter=family={{ openshift_gcp_target_image }}"
+        --format=json --sort-by ~creationTimestamp
+    register: images
+  - name: Prune oldest images
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images delete "{{ item['name'] }}"
+    with_items: "{{ (images.stdout | default('[]') | from_json )[( openshift_gcp_keep_images | int ):] }}"
+  when: openshift_gcp_keep_images is defined
+
+- name: Copy the latest image in the family {{ openshift_gcp_image }} to {{ openshift_gcp_target_image }}
+  command: >
+    gcloud --project "{{ openshift_gcp_target_project | default(openshift_gcp_project) }}"
+      beta compute images create
+      "{{ openshift_gcp_target_image_name | default(openshift_gcp_target_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}"
+      --family "{{ openshift_gcp_target_image }}"
+      --source-image-family "{{ openshift_gcp_image }}"
+      --source-image-project "{{ openshift_gcp_project }}"
diff --git a/roles/openshift_gcp/tasks/setup_scale_group_facts.yml b/roles/openshift_gcp/tasks/setup_scale_group_facts.yml
new file mode 100644
index 000000000..0fda43123
--- /dev/null
+++ b/roles/openshift_gcp/tasks/setup_scale_group_facts.yml
@@ -0,0 +1,44 @@
+---
+- name: Add masters to requisite groups
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: masters, etcd
+  with_items: "{{ groups['tag_ocp-master'] }}"
+
+- name: Add a master to the primary masters group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: primary_master
+  with_items: "{{ groups['tag_ocp-master'].0 }}"
+
+- name: Add non-bootstrapping master node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: infra
+  with_items: "{{ groups['tag_ocp-master'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add infra node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: infra
+  with_items: "{{ groups['tag_ocp-infra-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: app
+  with_items: "{{ groups['tag_ocp-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add bootstrap node instances
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: bootstrap_nodes
+    openshift_node_bootstrap: True
+  with_items: "{{ groups['tag_ocp-node'] | default([]) | intersect(groups['tag_ocp-bootstrap'] | default([])) }}"
+  when: not (openshift_node_bootstrap | default(False))
diff --git a/roles/openshift_gcp/templates/inventory.j2.sh b/roles/openshift_gcp/templates/inventory.j2.sh
new file mode 100644
index 000000000..dcaffb578
--- /dev/null
+++ b/roles/openshift_gcp/templates/inventory.j2.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+export GCE_PROJECT="{{ openshift_gcp_project }}"
+export GCE_ZONE="{{ openshift_gcp_zone }}"
+export GCE_EMAIL="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+export GCE_PEM_FILE_PATH="/tmp/gce.pem"
+export INVENTORY_IP_TYPE="{{ inventory_ip_type }}"
+export GCE_TAGGED_INSTANCES="{{ openshift_gcp_prefix }}ocp"
\ No newline at end of file
diff --git a/roles/openshift_gcp/templates/master_healthcheck.j2 b/roles/openshift_gcp/templates/master_healthcheck.j2
new file mode 100644
index 000000000..189e578c5
--- /dev/null
+++ b/roles/openshift_gcp/templates/master_healthcheck.j2
@@ -0,0 +1,68 @@
+#---------------------------------------------------------------------
+# Example configuration for a possible web application.  See the
+# full configuration options online.
+#
+#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
+#
+#---------------------------------------------------------------------
+
+#---------------------------------------------------------------------
+# Global settings
+#---------------------------------------------------------------------
+global
+    # to have these messages end up in /var/log/haproxy.log you will
+    # need to:
+    #
+    # 1) configure syslog to accept network log events.  This is done
+    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
+    #    /etc/sysconfig/syslog
+    #
+    # 2) configure local2 events to go to the /var/log/haproxy.log
+    #   file. A line like the following can be added to
+    #   /etc/sysconfig/syslog
+    #
+    #    local2.*                       /var/log/haproxy.log
+    #
+    log         127.0.0.1 local2
+
+    chroot      /var/lib/haproxy
+    pidfile     /var/run/haproxy.pid
+    maxconn     4000
+    user        haproxy
+    group       haproxy
+    daemon
+
+    # turn on stats unix socket
+    stats socket /var/lib/haproxy/stats
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 3
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           10s
+    maxconn                 3000
+
+#---------------------------------------------------------------------
+# main frontend which proxys to the backends
+#---------------------------------------------------------------------
+frontend  http-proxy *:8080
+    acl          url_healthz  path_beg  -i /healthz
+    use_backend  ocp          if url_healthz
+
+backend ocp
+    server       ocp localhost:{{ internal_console_port }} ssl verify none
diff --git a/roles/openshift_gcp/templates/openshift-bootstrap-update.j2 b/roles/openshift_gcp/templates/openshift-bootstrap-update.j2
new file mode 100644
index 000000000..5b0563724
--- /dev/null
+++ b/roles/openshift_gcp/templates/openshift-bootstrap-update.j2
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+oc serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
+gcloud compute project-info --project '{{ openshift_gcp_project }}' add-metadata --metadata-from-file '{{ openshift_gcp_prefix + openshift_gcp_clusterid | default("default") }}-bootstrap-config=/root/bootstrap.kubeconfig'
+rm -f /root/bootstrap.kubeconfig
diff --git a/roles/openshift_gcp/templates/provision.j2.sh b/roles/openshift_gcp/templates/provision.j2.sh
index 4d150bc74..794985322 100644
--- a/roles/openshift_gcp/templates/provision.j2.sh
+++ b/roles/openshift_gcp/templates/provision.j2.sh
@@ -9,15 +9,26 @@ if [[ -n "{{ openshift_gcp_ssh_private_key }}" ]]; then
         ssh-add "{{ openshift_gcp_ssh_private_key }}" || true
     fi
 
-    # Check if the ~/.ssh/google_compute_engine.pub key is in the project metadata, and if not, add it there
-    pub_key=$(cut -d ' ' -f 2 < "{{ openshift_gcp_ssh_private_key }}.pub")
+    # Check if the public key is in the project metadata, and if not, add it there
+    if [ -f "{{ openshift_gcp_ssh_private_key }}.pub" ]; then
+        pub_file="{{ openshift_gcp_ssh_private_key }}.pub"
+        pub_key=$(cut -d ' ' -f 2 < "{{ openshift_gcp_ssh_private_key }}.pub")
+    else
+        keyfile="${HOME}/.ssh/google_compute_engine"
+        pub_file="${keyfile}.pub"
+        mkdir -p "${HOME}/.ssh"
+        cp "{{ openshift_gcp_ssh_private_key }}" "${keyfile}"
+        chmod 0600 "${keyfile}"
+        ssh-keygen -y -f "${keyfile}" >  "${pub_file}"
+        pub_key=$(cut -d ' ' -f 2 <  "${pub_file}")
+    fi
     key_tmp_file='/tmp/ocp-gce-keys'
     if ! gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep -q "$pub_key"; then
         if gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep -q ssh-rsa; then
             gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep ssh-rsa | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/value: //' > "$key_tmp_file"
         fi
         echo -n 'cloud-user:' >> "$key_tmp_file"
-        cat "{{ openshift_gcp_ssh_private_key }}.pub" >> "$key_tmp_file"
+        cat "${pub_file}" >> "$key_tmp_file"
         gcloud --project "{{ openshift_gcp_project }}" compute project-info add-metadata --metadata-from-file "sshKeys=${key_tmp_file}"
         rm -f "$key_tmp_file"
     fi
diff --git a/roles/openshift_gcp/templates/yum_repo.j2 b/roles/openshift_gcp/templates/yum_repo.j2
new file mode 100644
index 000000000..77919ea75
--- /dev/null
+++ b/roles/openshift_gcp/templates/yum_repo.j2
@@ -0,0 +1,20 @@
+{% for repo in provision_custom_repositories %}
+[{{ repo.id | default(repo.name) }}]
+name={{ repo.name | default(repo.id) }}
+baseurl={{ repo.baseurl }}
+{% set enable_repo = repo.enabled | default(1) %}
+enabled={{ 1 if ( enable_repo == 1 or enable_repo == True ) else 0 }}
+{% set enable_gpg_check = repo.gpgcheck | default(1) %}
+gpgcheck={{ 1 if ( enable_gpg_check == 1 or enable_gpg_check == True ) else 0 }}
+{% if 'sslclientcert' in repo %}
+sslclientcert={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_cert" if repo.sslclientcert }}
+{% endif %}
+{% if 'sslclientkey' in repo %}
+sslclientkey={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_key" if repo.sslclientkey }}
+{% endif %}
+{% for key, value in repo.iteritems() %}
+{% if key not in ['id', 'name', 'baseurl', 'enabled', 'gpgcheck', 'sslclientkey', 'sslclientcert'] and value is defined %}
+{{ key }}={{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}