Infra node support

15 files changed, 145 insertions, 22 deletions

diff --git a/bin/cluster b/bin/cluster
index 746c0349a..7eb4a4448 100755
--- a/bin/cluster
+++ b/bin/cluster
@@ -51,6 +51,7 @@ class Cluster(object):
 
         env['num_masters'] = args.masters
         env['num_nodes'] = args.nodes
+        env['num_infra'] = args.infra
         env['num_etcd'] = args.etcd
 
         return self.action(args, inventory, env, playbook)
@@ -149,7 +150,7 @@ class Cluster(object):
 
             boto_conf_files = ['~/.aws/credentials', '~/.boto']
             conf_exists = lambda conf: os.path.isfile(os.path.expanduser(conf))
-            boto_configs = [ conf for conf in boto_conf_files if conf_exists(conf)]
+            boto_configs = [conf for conf in boto_conf_files if conf_exists(conf)]
 
             if len(key_missing) > 0 and len(boto_configs) == 0:
                 raise ValueError("PROVIDER aws requires {} environment variable(s). See README_AWS.md".format(missing))
@@ -262,6 +263,8 @@ if __name__ == '__main__':
                                help='number of masters to create in cluster')
     create_parser.add_argument('-n', '--nodes', default=2, type=int,
                                help='number of nodes to create in cluster')
+    create_parser.add_argument('-i', '--infra', default=1, type=int,
+                               help='number of infra nodes to create in cluster')
     create_parser.add_argument('-e', '--etcd', default=0, type=int,
                                help='number of external etcd hosts to create in cluster')
     create_parser.set_defaults(func=cluster.create)
diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index 18229631c..88e86f67c 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -50,7 +50,6 @@ class FilterModule(object):
 
         return [item for sublist in data for item in sublist]
 
-
     @staticmethod
     def oo_collect(data, attribute=None, filters=None):
         ''' This takes a list of dict and collects all attributes specified into a
diff --git a/playbooks/aws/openshift-cluster/launch.yml b/playbooks/aws/openshift-cluster/launch.yml
index 5db87fa90..a89275597 100644
--- a/playbooks/aws/openshift-cluster/launch.yml
+++ b/playbooks/aws/openshift-cluster/launch.yml
@@ -17,6 +17,7 @@
       instances: "{{ etcd_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "default"
 
   - include: ../../common/openshift-cluster/set_master_launch_facts_tasks.yml
   - include: tasks/launch_instances.yml
@@ -24,13 +25,29 @@
       instances: "{{ master_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "default"
 
   - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "compute"
+      count: "{{ num_nodes }}"
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ node_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
+
+  - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "infra"
+      count: "{{ num_infra }}"
   - include: tasks/launch_instances.yml
     vars:
       instances: "{{ node_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
 
   - add_host:
       name: "{{ master_names.0 }}"
diff --git a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
index 25a87aaf6..92155582e 100644
--- a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
@@ -5,6 +5,7 @@
     env: "{{ cluster }}"
     env_host_type: "{{ cluster }}-openshift-{{ type }}"
     host_type: "{{ type }}"
+    sub_host_type: "{{ g_sub_host_type }}"
 
 - set_fact:
     ec2_region: "{{ lookup('env', 'ec2_region')
@@ -34,6 +35,35 @@
     ec2_assign_public_ip: "{{ lookup('env', 'ec2_assign_public_ip')
                     | default(deployment_vars[deployment_type].assign_public_ip, true) }}"
   when: ec2_assign_public_ip is not defined
+
+- set_fact:
+    ec2_instance_type: "{{ ec2_master_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_security_groups: "{{ ec2_master_security_groups
+                    | default(deployment_vars[deployment_type].security_groups, true) }}"
+  when: host_type == "master" and sub_host_type == "default"
+
+- set_fact:
+    ec2_instance_type: "{{ ec2_etcd_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_security_groups: "{{ ec2_etcd_security_groups
+                    | default(deployment_vars[deployment_type].security_groups, true)}}"
+  when: host_type == "etcd" and sub_host_type == "default"
+
+- set_fact:
+    ec2_instance_type: "{{ ec2_infra_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_security_groups: "{{ ec2_infra_security_groups
+                    | default(deployment_vars[deployment_type].security_groups, true) }}"
+  when: host_type == "node" and sub_host_type == "infra"
+
+- set_fact:
+    ec2_instance_type: "{{ ec2_node_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_security_groups: "{{ ec2_node_security_groups
+                    | default(deployment_vars[deployment_type].security_groups, true) }}"
+  when: host_type == "node" and sub_host_type == "compute"
+
+- set_fact:
+    ec2_instance_type: "{{ lookup('env', 'ec2_instance_type')
+                          | default(deployment_vars[deployment_type].type, true) }}"
+  when: ec2_instance_type is not defined
 - set_fact:
     ec2_security_groups: "{{ lookup('env', 'ec2_security_groups')
                     | default(deployment_vars[deployment_type].security_groups, true) }}"
@@ -99,6 +129,7 @@
       env: "{{ env }}"
       host-type: "{{ host_type }}"
       env-host-type: "{{ env_host_type }}"
+      sub-host-type: "{{ sub_host_type }}"
     volumes: "{{ volumes }}"
   register: ec2
 
@@ -112,7 +143,9 @@
       Name: "{{ item.0 }}"
 
 - set_fact:
-    instance_groups: tag_created-by_{{ created_by }}, tag_env_{{ env }}, tag_host-type_{{ host_type }}, tag_env-host-type_{{ env_host_type }}
+    instance_groups: "tag_created-by_{{ created_by }}, tag_env_{{ env }},
+                    tag_host-type_{{ host_type }}, tag_env-host-type_{{ env_host_type }},
+                    tag_sub-host-type_{{ sub_host_type }}"
 
 - name: Add new instances groups and variables
   add_host:
diff --git a/playbooks/aws/openshift-cluster/terminate.yml b/playbooks/aws/openshift-cluster/terminate.yml
index 3a08ed966..77287cad0 100644
--- a/playbooks/aws/openshift-cluster/terminate.yml
+++ b/playbooks/aws/openshift-cluster/terminate.yml
@@ -37,6 +37,7 @@
           env: "{{ item['ec2_tag_env'] }}"
           host-type: "{{ item['ec2_tag_host-type'] }}"
           env-host-type: "{{ item['ec2_tag_env-host-type'] }}"
+          sub_host_type: "{{ item['ec2_tag_sub-host-type'] }}"
       with_items: host_vars
       when: "'oo_hosts_to_terminate' in groups"
 
diff --git a/playbooks/aws/openshift-cluster/vars.online.int.yml b/playbooks/aws/openshift-cluster/vars.online.int.yml
index e406a7635..fc8b8d2d2 100644
--- a/playbooks/aws/openshift-cluster/vars.online.int.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.int.yml
@@ -3,7 +3,13 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_instance_type: m3.large
-ec2_security_groups: [ 'int-v3' ]
+ec2_master_instance_type: m3.large
+ec2_master_security_groups: [ 'integration', 'integration-master' ]
+ec2_infra_instance_type: m3.large
+ec2_infra_security_groups: [ 'integration', 'integration-infra' ]
+ec2_node_instance_type: m3.large
+ec2_node_security_groups: [ 'integration', 'integration-node' ]
+ec2_etcd_instance_type: m3.large
+ec2_etcd_security_groups: [ 'integration', 'integration-etcd' ]
 ec2_vpc_subnet: subnet-987c0def
 ec2_assign_public_ip: yes
diff --git a/playbooks/aws/openshift-cluster/vars.online.prod.yml b/playbooks/aws/openshift-cluster/vars.online.prod.yml
index e406a7635..f68d41fc4 100644
--- a/playbooks/aws/openshift-cluster/vars.online.prod.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.prod.yml
@@ -3,7 +3,13 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_instance_type: m3.large
-ec2_security_groups: [ 'int-v3' ]
+ec2_master_instance_type: m3.large
+ec2_master_security_groups: [ 'production', 'production-master' ]
+ec2_infra_instance_type: m3.large
+ec2_infra_security_groups: [ 'production', 'production-infra' ]
+ec2_node_instance_type: m3.large
+ec2_node_security_groups: [ 'production', 'production-node' ]
+ec2_etcd_instance_type: m3.large
+ec2_etcd_security_groups: [ 'production', 'production-etcd' ]
 ec2_vpc_subnet: subnet-987c0def
 ec2_assign_public_ip: yes
diff --git a/playbooks/aws/openshift-cluster/vars.online.stage.yml b/playbooks/aws/openshift-cluster/vars.online.stage.yml
index e406a7635..ce9869fcd 100644
--- a/playbooks/aws/openshift-cluster/vars.online.stage.yml
+++ b/playbooks/aws/openshift-cluster/vars.online.stage.yml
@@ -3,7 +3,13 @@ ec2_image: ami-9101c8fa
 ec2_image_name: libra-ops-rhel7*
 ec2_region: us-east-1
 ec2_keypair: mmcgrath_libra
-ec2_instance_type: m3.large
-ec2_security_groups: [ 'int-v3' ]
+ec2_master_instance_type: m3.large
+ec2_master_security_groups: [ 'stage', 'stage-master' ]
+ec2_infra_instance_type: m3.large
+ec2_infra_security_groups: [ 'stage', 'stage-infra' ]
+ec2_node_instance_type: m3.large
+ec2_node_security_groups: [ 'stage', 'stage-node' ]
+ec2_etcd_instance_type: m3.large
+ec2_etcd_security_groups: [ 'stage', 'stage-etcd' ]
 ec2_vpc_subnet: subnet-987c0def
 ec2_assign_public_ip: yes
diff --git a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
index 96e1a9a63..278942f8b 100644
--- a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
+++ b/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
@@ -1,11 +1,13 @@
 ---
-- set_fact: k8s_type="node"
+- set_fact: k8s_type=node
+- set_fact: sub_host_type="{{ type }}"
+- set_fact: number_nodes="{{ count }}"
 
 - name: Generate node instance names(s)
   set_fact:
-    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
+    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ sub_host_type }}-{{ '%05x' | format(1048576 | random) }}"
   register: node_names_output
-  with_sequence: count={{ num_nodes }}
+  with_sequence: count={{ number_nodes }}
 
 - set_fact:
     node_names: "{{ node_names_output.results | default([])
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index bd35008b8..6ef375bbb 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -129,16 +129,14 @@
     openshift_nodes: "{{ hostvars
                          | oo_select_keys(groups['oo_nodes_to_config'])
                          | oo_collect('openshift.common.hostname') }}"
-    openshift_unscheduleable_nodes: "{{ hostvars
-                                        | oo_select_keys(groups['oo_nodes_to_config']
-                                                         | default([]))
-                                        | oo_collect('openshift.common.hostname', {'openshift_scheduleable': False}) }}"
+    openshift_unscheduleable_nodes: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] | default([]))
+                                      | oo_collect('openshift.common.hostname', {'openshift_scheduleable': False}) }}"
   pre_tasks:
   - set_fact:
       openshift_scheduleable_nodes: "{{ hostvars
-                                      | oo_select_keys(groups['oo_nodes_to_config']
-                                                       | default([]))
+                                      | oo_select_keys(groups['oo_nodes_to_config'] | default([]))
                                       | oo_collect('openshift.common.hostname')
                                       | difference(openshift_unscheduleable_nodes) }}"
+
   roles:
   - openshift_manage_node
diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml
index 35737f03d..7a3b80da0 100644
--- a/playbooks/gce/openshift-cluster/launch.yml
+++ b/playbooks/gce/openshift-cluster/launch.yml
@@ -15,17 +15,33 @@
       instances: "{{ master_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "default"
 
   - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "compute"
+      count: "{{ num_nodes }}"
   - include: tasks/launch_instances.yml
     vars:
       instances: "{{ node_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
+
+  - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "infra"
+      count: "{{ num_infra }}"
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ infra_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
 
   - set_fact:
-      a_master: "{{ master_names[0] }}"
-  - add_host: name={{ a_master }} groups=service_master
+      a_infra: "{{ infra_names[0] }}"
+  - add_host: name={{ a_infra }} groups=service_master
 
 - include: update.yml
 
diff --git a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
index 9a9848f05..6307ecc27 100644
--- a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
@@ -14,6 +14,7 @@
       - created-by-{{ lookup('env', 'LOGNAME') |default(cluster, true) }}
       - env-{{ cluster }}
       - host-type-{{ type }}
+      - sub-host-type-{{ sub_host_type }}
       - env-host-type-{{ cluster }}-openshift-{{ type }}
   register: gce
 
diff --git a/playbooks/libvirt/openshift-cluster/launch.yml b/playbooks/libvirt/openshift-cluster/launch.yml
index 6badcb325..6630fa27d 100644
--- a/playbooks/libvirt/openshift-cluster/launch.yml
+++ b/playbooks/libvirt/openshift-cluster/launch.yml
@@ -23,13 +23,29 @@
       instances: "{{ master_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "default"
 
   - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "compute"
+      count: "{{ num_nodes }}"
   - include: tasks/launch_instances.yml
     vars:
       instances: "{{ node_names }}"
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
+
+  - include: ../../common/openshift-cluster/set_node_launch_facts_tasks.yml
+    vars:
+      type: "infra"
+      count: "{{ num_infra }}"
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ infra_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
 
 - include: update.yml
 
diff --git a/playbooks/openstack/openshift-cluster/launch.yml b/playbooks/openstack/openshift-cluster/launch.yml
index 3cdd2ae4d..d41448dc0 100644
--- a/playbooks/openstack/openshift-cluster/launch.yml
+++ b/playbooks/openstack/openshift-cluster/launch.yml
@@ -27,10 +27,13 @@
              -P ssh_incoming={{ openstack_ssh_access_from }}
              -P num_masters={{ num_masters }}
              -P num_nodes={{ num_nodes }}
+             -P num_infra={{ num_infra }}
              -P master_image={{ deployment_vars[deployment_type].image }}
              -P node_image={{ deployment_vars[deployment_type].image }}
+             -P infra_image={{ deployment_vars[deployment_type].image }}
              -P master_flavor={{ openstack_flavor["master"] }}
              -P node_flavor={{ openstack_flavor["node"] }}
+             -P infra_flavor={{ openstack_flavor["infra"] }}
              -P ssh_public_key="{{ openstack_ssh_public_key }}"
              openshift-ansible-{{ cluster_id }}-stack'
     when: stack_show_result.rc == 1
@@ -43,10 +46,13 @@
              -P ssh_incoming={{ openstack_ssh_access_from }}
              -P num_masters={{ num_masters }}
              -P num_nodes={{ num_nodes }}
+             -P num_infra={{ num_infra }}
              -P master_image={{ deployment_vars[deployment_type].image }}
              -P node_image={{ deployment_vars[deployment_type].image }}
+             -P infra_image={{ deployment_vars[deployment_type].image }}
              -P master_flavor={{ openstack_flavor["master"] }}
              -P node_flavor={{ openstack_flavor["node"] }}
+             -P infra_flavor={{ openstack_flavor["infra"] }}
              -P ssh_public_key="{{ openstack_ssh_public_key }}"
              openshift-ansible-{{ cluster_id }}-stack'
     when: stack_show_result.rc == 0
@@ -72,7 +78,7 @@
       ansible_ssh_host: '{{ item[2] }}'
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: 'tag_env_{{ cluster_id }}, tag_host-type_master, tag_env-host-type_{{ cluster_id }}-openshift-master'
+      groups: 'tag_env_{{ cluster_id }}, tag_host-type_master, tag_env-host-type_{{ cluster_id }}-openshift-master, tag_sub-host-type_default'
     with_together:
       - parsed_outputs.master_names
       - parsed_outputs.master_ips
@@ -84,12 +90,24 @@
       ansible_ssh_host: '{{ item[2] }}'
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-      groups: 'tag_env_{{ cluster_id }}, tag_host-type_node, tag_env-host-type_{{ cluster_id }}-openshift-node'
+      groups: 'tag_env_{{ cluster_id }}, tag_host-type_node, tag_env-host-type_{{ cluster_id }}-openshift-node, tag_sub-host-type_node'
     with_together:
       - parsed_outputs.node_names
       - parsed_outputs.node_ips
       - parsed_outputs.node_floating_ips
 
+  - name: Add new infra instances groups and variables
+    add_host:
+      hostname: '{{ item[0] }}'
+      ansible_ssh_host: '{{ item[2] }}'
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: 'tag_env_{{ cluster_id }}, tag_host-type_node, tag_env-host-type_{{ cluster_id }}-openshift-node, tag_sub-host-type_infra'
+    with_together:
+      - parsed_outputs.infra_names
+      - parsed_outputs.infra_ips
+      - parsed_outputs.infra_floating_ips
+
   - name: Wait for ssh
     wait_for:
       host: '{{ item }}'
diff --git a/playbooks/openstack/openshift-cluster/vars.yml b/playbooks/openstack/openshift-cluster/vars.yml
index d077a6ced..43e25f2e6 100644
--- a/playbooks/openstack/openshift-cluster/vars.yml
+++ b/playbooks/openstack/openshift-cluster/vars.yml
@@ -19,6 +19,7 @@ openstack_ssh_access_from:      "{{ lookup('oo_option', 'ssh_from')          |
                                     default('0.0.0.0/0',                     True) }}"
 openstack_flavor:
   master: "{{ lookup('oo_option', 'master_flavor'    ) | default('m1.small',  True) }}"
+  infra:  "{{ lookup('oo_option', 'infra_flavor'     ) | default('m1.small',  True) }}"
   node:   "{{ lookup('oo_option', 'node_flavor'      ) | default('m1.medium', True) }}"
 
 deployment_vars: