fixed some issues to boot VM on GCE

corrected openshift master config, commented infra

correct list and terminate, it was bugged in case where no instance were terminated

Using openshift-sdn for gce

new join_node playbook for gce

openstack/hosts/nova.py is now taking the nova/ini of its directory and not the directory of execution of bin/cluster

add fix of ICMP reject rules

Avoid a recursive loop

Jenkins image was renamed

Default masters to t2.medium instead of t2.small

Fix a minor bug involving AWS ENV Keys

* If a user forgot to set their AWS keys, we'd get a non descriptive error about a variable not being set
* This patch uses the correct variable so the error message is more informative

delete some fix that are needed anymore (selinux, iptables rules for sdn)

GCE : all variables needed are in gce.ini, it will be used by bin/cluster (now check better the presence of gce.init in the default place or use GCE_INI_PATH to locate it ), also by gce.ini

openshift_node_labels : get from oo_option

fix syntax error in bin/cluster

fix lookup for openshift_node_labels

Adding desc, multiplier, and units to zabbix item

Adding capability to have descriptions on triggers

updated triggers and items to have better descriptions and multipliers

Move openshift_data_dir to a fact based on deployment_type

Previously this was being set to /var/lib/origin regardless of deployment_type
which isn't correct given that existing 'enterprise' and 'online' deployments
would have been deployed with /var/lib/openshift

Verify again that ansible version is different than 1.9.0 and 1.9.0.1

bin/cluste does not take -a and -s anymore

fix master_public_api_url : using by default a correct url

Really fixed master public api url this time

Really fixed master public api url this time

uncommented infra deployment like before

fixed again masterpublicurl in a template

README_GCE.md : use GCE_INI_PATH in order to locate gce.ini, update description of gce.ini

13 files changed, 184 insertions, 43 deletions

diff --git a/playbooks/aws/openshift-cluster/vars.online.int.yml b/playbooks/aws/openshift-cluster/vars.online.int.yml
index bb18e13b0..2e2f25ccd 100644
--- a/playbooks/aws/openshift-cluster/vars.online.int.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.int.yml
@@ -3,7 +3,7 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_master_instance_type: t2.small
+ec2_master_instance_type: t2.medium
 ec2_master_security_groups: [ 'integration', 'integration-master' ]
 ec2_infra_instance_type: c4.large
 ec2_infra_security_groups: [ 'integration', 'integration-infra' ]
diff --git a/playbooks/aws/openshift-cluster/vars.online.prod.yml b/playbooks/aws/openshift-cluster/vars.online.prod.yml
index bbef9cc56..18a53e12e 100644
--- a/playbooks/aws/openshift-cluster/vars.online.prod.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.prod.yml
@@ -3,7 +3,7 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_master_instance_type: t2.small
+ec2_master_instance_type: t2.medium
 ec2_master_security_groups: [ 'production', 'production-master' ]
 ec2_infra_instance_type: c4.large
 ec2_infra_security_groups: [ 'production', 'production-infra' ]
diff --git a/playbooks/aws/openshift-cluster/vars.online.stage.yml b/playbooks/aws/openshift-cluster/vars.online.stage.yml
index 9008a55ba..1f9ac4252 100644
--- a/playbooks/aws/openshift-cluster/vars.online.stage.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.stage.yml
@@ -3,7 +3,7 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_master_instance_type: t2.small
+ec2_master_instance_type: t2.medium
 ec2_master_security_groups: [ 'stage', 'stage-master' ]
 ec2_infra_instance_type: c4.large
 ec2_infra_security_groups: [ 'stage', 'stage-infra' ]
diff --git a/playbooks/common/openshift-cluster/set_infra_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/set_infra_launch_facts_tasks.yml
new file mode 100644
index 000000000..0fd53eb7d
--- /dev/null
+++ b/playbooks/common/openshift-cluster/set_infra_launch_facts_tasks.yml
@@ -0,0 +1,15 @@
+---
+- set_fact: k8s_type=infra
+- set_fact: sub_host_type="{{ type }}"
+- set_fact: number_infra="{{ count }}"
+
+- name: Generate infra  instance names(s)
+  set_fact:
+    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ sub_host_type }}-{{ '%05x' | format(1048576 | random) }}"
+  register: infra_names_output
+  with_sequence: count={{ number_infra }}
+
+- set_fact:
+    infra_names: "{{ infra_names_output.results | default([])
+                    | oo_collect('ansible_facts')
+                    | oo_collect('scratch_name') }}"
diff --git a/playbooks/gce/openshift-cluster/config.yml b/playbooks/gce/openshift-cluster/config.yml
index fd5dfcc72..7bd3f1a56 100644
--- a/playbooks/gce/openshift-cluster/config.yml
+++ b/playbooks/gce/openshift-cluster/config.yml
@@ -10,6 +10,8 @@
   - set_fact:
       g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
       g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
+      use_sdn: "{{ do_we_use_openshift_sdn }}"
+      sdn_plugin: "{{ sdn_network_plugin }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
@@ -22,3 +24,5 @@
     openshift_debug_level: 2
     openshift_deployment_type: "{{ deployment_type }}"
     openshift_hostname: "{{ gce_private_ip }}"
+    openshift_use_openshift_sdn: "{{ hostvars.localhost.use_sdn  }}"
+    os_sdn_network_plugin_name: "{{ hostvars.localhost.sdn_plugin }}"
diff --git a/playbooks/gce/openshift-cluster/join_node.yml b/playbooks/gce/openshift-cluster/join_node.yml
new file mode 100644
index 000000000..613bbb34f
--- /dev/null
+++ b/playbooks/gce/openshift-cluster/join_node.yml
@@ -0,0 +1,64 @@
+---
+- name: Populate oo_hosts_to_update group
+  hosts: localhost
+  gather_facts: no
+  vars_files:
+  - vars.yml
+  tasks:
+  - name: Evaluate oo_hosts_to_update
+    add_host:
+      name: "{{ node_ip }}"
+      groups: oo_hosts_to_update
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+
+- include: ../../common/openshift-cluster/update_repos_and_packages.yml
+
+- name: Populate oo_masters_to_config host group
+  hosts: localhost
+  gather_facts: no
+  vars_files:
+  - vars.yml
+  tasks:
+  - name: Evaluate oo_nodes_to_config
+    add_host:
+      name: "{{ node_ip }}"
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: oo_nodes_to_config
+
+  - name: Add to preemptible group if needed
+    add_host:
+      name: "{{ node_ip }}"
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: oo_preemptible_nodes
+    when: preemptible is defined and preemptible == "true"
+  
+  - name: Add to not preemptible group if needed
+    add_host:
+      name: "{{ node_ip }}"
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: oo_non_preemptible_nodes
+    when: preemptible is defined and  preemptible == "false"
+  
+  - name: Evaluate oo_first_master
+    add_host:
+      name: "{{ groups['tag_env-host-type-' ~ cluster_id ~ '-openshift-master'][0] }}"
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: oo_first_master
+    when: "'tag_env-host-type-{{ cluster_id }}-openshift-master' in groups"
+
+#- include: config.yml
+- include: ../../common/openshift-node/config.yml
+  vars:
+    openshift_cluster_id: "{{ cluster_id }}"
+    openshift_debug_level: 4
+    openshift_deployment_type: "{{ deployment_type }}"
+    openshift_hostname: "{{ ansible_default_ipv4.address }}"
+    openshift_use_openshift_sdn: true
+    os_sdn_network_plugin_name: "redhat/openshift-ovs-subnet"
+    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
+    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml
index 7a3b80da0..762fa9e8d 100644
--- a/playbooks/gce/openshift-cluster/launch.yml
+++ b/playbooks/gce/openshift-cluster/launch.yml
@@ -28,7 +28,7 @@
       type: "{{ k8s_type }}"
       g_sub_host_type: "{{ sub_host_type }}"
 
-  - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+  - include: ../../common/openshift-cluster/set_infra_launch_facts_tasks.yml
     vars:
       type: "infra"
       count: "{{ num_infra }}"
diff --git a/playbooks/gce/openshift-cluster/list.yml b/playbooks/gce/openshift-cluster/list.yml
index 5ba0f5a48..f5f89baf0 100644
--- a/playbooks/gce/openshift-cluster/list.yml
+++ b/playbooks/gce/openshift-cluster/list.yml
@@ -14,11 +14,11 @@
       groups: oo_list_hosts
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated)
+    with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated | default([]))
 
 - name: List instance(s)
   hosts: oo_list_hosts
   gather_facts: no
   tasks:
   - debug:
-      msg: "public ip:{{ hostvars[inventory_hostname].gce_public_ip }} private ip:{{ hostvars[inventory_hostname].gce_private_ip }}"
+      msg: "private ip:{{ hostvars[inventory_hostname].gce_private_ip }}"
diff --git a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
index 6307ecc27..f569b2a37 100644
--- a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
@@ -10,18 +10,22 @@
     service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
     pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
     project_id: "{{ lookup('env', 'gce_project_id') }}"
+    zone: "{{ lookup('env', 'zone') }}"
+    network: "{{ lookup('env', 'network') }}"
+# unsupported in 1.9.+
+    #service_account_permissions: "datastore,logging-write"
     tags:
       - created-by-{{ lookup('env', 'LOGNAME') |default(cluster, true) }}
       - env-{{ cluster }}
       - host-type-{{ type }}
-      - sub-host-type-{{ sub_host_type }}
+      - sub-host-type-{{ g_sub_host_type }}
       - env-host-type-{{ cluster }}-openshift-{{ type }}
   register: gce
 
 - name: Add new instances to groups and set variables needed
   add_host:
     hostname: "{{ item.name }}"
-    ansible_ssh_host: "{{ item.public_ip }}"
+    ansible_ssh_host: "{{ item.name }}"
     ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
     ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
     groups: "{{ item.tags | oo_prepend_strings_in_list('tag_') | join(',') }}"
@@ -30,13 +34,13 @@
   with_items: gce.instance_data
 
 - name: Wait for ssh
-  wait_for: port=22 host={{ item.public_ip }}
+  wait_for: port=22 host={{ item.name }}
   with_items: gce.instance_data
 
 - name: Wait for user setup
   command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null {{ hostvars[item.name].ansible_ssh_user }}@{{ item.public_ip }} echo {{ hostvars[item.name].ansible_ssh_user }} user is setup"
   register: result
   until: result.rc == 0
-  retries: 20
-  delay: 10
+  retries: 30
+  delay: 5
   with_items: gce.instance_data
diff --git a/playbooks/gce/openshift-cluster/terminate.yml b/playbooks/gce/openshift-cluster/terminate.yml
index 098b0df73..f705745d9 100644
--- a/playbooks/gce/openshift-cluster/terminate.yml
+++ b/playbooks/gce/openshift-cluster/terminate.yml
@@ -1,25 +1,18 @@
 ---
 - name: Terminate instance(s)
   hosts: localhost
+  connection: local
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - set_fact: scratch_group=tag_env-host-type-{{ cluster_id }}-openshift-node
+  - set_fact: scratch_group=tag_env-{{ cluster_id }}
   - add_host:
       name: "{{ item }}"
-      groups: oo_hosts_to_terminate, oo_nodes_to_terminate
+      groups: oo_hosts_to_terminate
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated)
-
-  - set_fact: scratch_group=tag_env-host-type-{{ cluster_id }}-openshift-master
-  - add_host:
-      name: "{{ item }}"
-      groups: oo_hosts_to_terminate, oo_masters_to_terminate
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user | default(ansible_ssh_user, true) }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated)
+    with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated | default([]))
 
 - name: Unsubscribe VMs
   hosts: oo_hosts_to_terminate
@@ -32,14 +25,34 @@
           lookup('oo_option', 'rhel_skip_subscription') | default(rhsub_skip, True) |
             default('no', True) | lower in ['no', 'false']
 
-- include: ../openshift-node/terminate.yml
-  vars:
-    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
-    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
-    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
+- name: Terminate instances(s)
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars_files:
+  - vars.yml
+  tasks:
+
+    - name: Terminate instances that were previously launched
+      local_action:
+        module: gce
+        state: 'absent'
+        name: "{{ item }}"
+        service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+        pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+        project_id: "{{ lookup('env', 'gce_project_id') }}"
+        zone: "{{ lookup('env', 'zone') }}"
+      with_items: groups['oo_hosts_to_terminate'] | default([])
+      when: item is defined
 
-- include: ../openshift-master/terminate.yml
-  vars:
-    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
-    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
-    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
+#- include: ../openshift-node/terminate.yml
+#  vars:
+#    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+#    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+#    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
+#
+#- include: ../openshift-master/terminate.yml
+#  vars:
+#    gce_service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
+#    gce_pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
+#    gce_project_id: "{{ lookup('env', 'gce_project_id') }}"
diff --git a/playbooks/gce/openshift-cluster/vars.yml b/playbooks/gce/openshift-cluster/vars.yml
index ae33083b9..6de007807 100644
--- a/playbooks/gce/openshift-cluster/vars.yml
+++ b/playbooks/gce/openshift-cluster/vars.yml
@@ -1,8 +1,11 @@
 ---
+do_we_use_openshift_sdn: true
+sdn_network_plugin: redhat/openshift-ovs-subnet 
+# os_sdn_network_plugin_name can be ovssubnet or multitenant, see https://docs.openshift.org/latest/architecture/additional_concepts/sdn.html#ovssubnet-plugin-operation
 deployment_vars:
   origin:
-    image: centos-7
-    ssh_user:
+    image: preinstalled-slave-50g-v5
+    ssh_user: root
     sudo: yes
   online:
     image: libra-rhel7
@@ -12,4 +15,3 @@ deployment_vars:
     image: rhel-7
     ssh_user:
     sudo: yes
-
diff --git a/playbooks/openstack/openshift-cluster/files/heat_stack.yaml b/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
index 40e4ab22c..e3e2b6872 100644
--- a/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
+++ b/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
@@ -88,6 +88,12 @@ parameters:
     label: Infra flavor
     description: Flavor of the infra node servers
 
+  key_pair:
+    type: string
+    label: Key name
+    description: Name of the key
+
+
 outputs:
 
   master_names:
@@ -250,6 +256,14 @@ resources:
           port_range_max: 10250
           remote_mode: remote_group_id
           remote_group_id: { get_resource: master-secgrp }
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 30001 
+          port_range_max: 30001
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 30850 
+          port_range_max: 30850
 
   infra-secgrp:
     type: OS::Neutron::SecurityGroup
@@ -291,7 +305,7 @@ resources:
           type:       master
           image:      { get_param: master_image }
           flavor:     { get_param: master_flavor }
-          key_name:   { get_resource: keypair }
+          key_name:   { get_param: key_pair }
           net:        { get_resource: net }
           subnet:     { get_resource: subnet }
           secgrp:
@@ -323,7 +337,7 @@ resources:
           subtype:    compute
           image:      { get_param: node_image }
           flavor:     { get_param: node_flavor }
-          key_name:   { get_resource: keypair }
+          key_name:   { get_param: key_pair }
           net:        { get_resource: net }
           subnet:     { get_resource: subnet }
           secgrp:
@@ -355,7 +369,7 @@ resources:
           subtype:    infra
           image:      { get_param: infra_image }
           flavor:     { get_param: infra_flavor }
-          key_name:   { get_resource: keypair }
+          key_name:   { get_param: key_pair }
           net:        { get_resource: net }
           subnet:     { get_resource: subnet }
           secgrp:
diff --git a/playbooks/openstack/openshift-cluster/launch.yml b/playbooks/openstack/openshift-cluster/launch.yml
index 651aef40b..5f1780476 100644
--- a/playbooks/openstack/openshift-cluster/launch.yml
+++ b/playbooks/openstack/openshift-cluster/launch.yml
@@ -19,15 +19,32 @@
     changed_when: false
     failed_when: stack_show_result.rc != 0 and 'Stack not found' not in stack_show_result.stderr
 
-  - set_fact:
-      heat_stack_action: 'stack-create'
+  - name: Create OpenStack Stack
+    command: 'heat stack-create -f {{ openstack_infra_heat_stack }}
+             -P key_pair={{ openstack_ssh_keypair }}
+             -P cluster_id={{ cluster_id }}
+             -P dns_nameservers={{ openstack_network_dns | join(",") }}
+             -P cidr={{ openstack_network_cidr }}
+             -P ssh_incoming={{ openstack_ssh_access_from }}
+             -P num_masters={{ num_masters }}
+             -P num_nodes={{ num_nodes }}
+             -P num_infra={{ num_infra }}
+             -P master_image={{ deployment_vars[deployment_type].image }}
+             -P node_image={{ deployment_vars[deployment_type].image }}
+             -P infra_image={{ deployment_vars[deployment_type].image }}
+             -P master_flavor={{ openstack_flavor["master"] }}
+             -P node_flavor={{ openstack_flavor["node"] }}
+             -P infra_flavor={{ openstack_flavor["infra"] }}
+             -P ssh_public_key="{{ openstack_ssh_public_key }}"
+             openshift-ansible-{{ cluster_id }}-stack'
     when: stack_show_result.rc == 1
   - set_fact:
       heat_stack_action: 'stack-update'
     when: stack_show_result.rc == 0
 
-  - name: Create or Update OpenStack Stack
-    command: 'heat {{ heat_stack_action }} -f {{ openstack_infra_heat_stack }}
+  - name: Update OpenStack Stack
+    command: 'heat stack-update -f {{ openstack_infra_heat_stack }}
+             -P key_pair={{ openstack_ssh_keypair }}
              -P cluster_id={{ cluster_id }}
              -P cidr={{ openstack_network_cidr }}
              -P dns_nameservers={{ openstack_network_dns | join(",") }}
@@ -50,7 +67,7 @@
     shell: 'heat stack-show openshift-ansible-{{ cluster_id }}-stack | awk ''$2 == "stack_status" {print $4}'''
     register: stack_show_status_result
     until: stack_show_status_result.stdout not in ['CREATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS']
-    retries: 30
+    retries: 300
     delay: 1
     failed_when: stack_show_status_result.stdout not in ['CREATE_COMPLETE', 'UPDATE_COMPLETE']
 
@@ -119,4 +136,12 @@
 
 - include: update.yml
 
+# Fix icmp reject iptables rules
+# It should be solved in openshift-sdn but unfortunately it's not the case
+# Mysterious
+- name: Configuring Nodes for RBox
+  hosts: oo_nodes_to_config
+  roles:
+    - rbox-node
+
 - include: list.yml