Merge pull request #6069 from kwoodson/upgrade_scale_groups

Initial upgrade for scale groups.

11 files changed, 160 insertions, 39 deletions

diff --git a/roles/openshift_aws/tasks/accept_nodes.yml b/roles/openshift_aws/tasks/accept_nodes.yml
new file mode 100644
index 000000000..ae320962f
--- /dev/null
+++ b/roles/openshift_aws/tasks/accept_nodes.yml
@@ -0,0 +1,34 @@
+---
+- name: fetch masters
+  ec2_remote_facts:
+    region: "{{ openshift_aws_region | default('us-east-1') }}"
+    filters:
+      "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid,
+           'tag:host-type': 'master', 'instance-state-name': 'running'} }}"
+  register: mastersout
+  retries: 20
+  delay: 3
+  until: "'instances' in mastersout and mastersout.instances|length > 0"
+
+- name: fetch new node instances
+  ec2_remote_facts:
+    region: "{{ openshift_aws_region }}"
+    filters:
+      "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid,
+           'tag:host-type': 'node',
+           'instance-state-name': 'running'} }}"
+  register: instancesout
+  retries: 20
+  delay: 3
+  until: "'instances' in instancesout and instancesout.instances|length > 0"
+
+- debug:
+    msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list }}"
+
+- name: approve nodes
+  oc_adm_csr:
+    #approve_all: True
+    nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list  }}"
+    timeout: 60
+  register: nodeout
+  delegate_to: "{{ mastersout.instances[0].public_ip_address }}"
diff --git a/roles/openshift_aws/tasks/elb.yml b/roles/openshift_aws/tasks/elb.yml
index a543222d5..5d371ec7a 100644
--- a/roles/openshift_aws/tasks/elb.yml
+++ b/roles/openshift_aws/tasks/elb.yml
@@ -17,6 +17,7 @@
     listeners: "{{ item.value }}"
     scheme: "{{ openshift_aws_elb_scheme }}"
     tags: "{{ openshift_aws_elb_tags }}"
+    wait: True
   register: new_elb
   with_dict: "{{ l_elb_dict_item.value }}"
 
diff --git a/roles/openshift_aws/tasks/launch_config_create.yml b/roles/openshift_aws/tasks/launch_config_create.yml
index a688496d2..f7f0f0953 100644
--- a/roles/openshift_aws/tasks/launch_config_create.yml
+++ b/roles/openshift_aws/tasks/launch_config_create.yml
@@ -10,7 +10,7 @@
 # Create the scale group config
 - name: Create the node scale group launch config
   ec2_lc:
-    name: "{{ openshift_aws_launch_config_basename }}-{{ launch_config_item.key }}-{{ l_epoch_time }}"
+    name: "{{ openshift_aws_launch_config_basename }}-{{ launch_config_item.key }}{{'-' ~ openshift_aws_new_version if openshift_aws_new_version != '' else '' }}"
     region: "{{ openshift_aws_region }}"
     image_id: "{{ l_aws_ami_map[launch_config_item.key] | default(openshift_aws_ami) }}"
     instance_type: "{{ launch_config_item.value.instance_type }}"
diff --git a/roles/openshift_aws/tasks/provision_nodes.yml b/roles/openshift_aws/tasks/provision_nodes.yml
index 1b40f24d3..3e84666a2 100644
--- a/roles/openshift_aws/tasks/provision_nodes.yml
+++ b/roles/openshift_aws/tasks/provision_nodes.yml
@@ -12,7 +12,9 @@
   register: instancesout
   retries: 20
   delay: 3
-  until: instancesout.instances|length > 0
+  until:
+  - "'instances' in instancesout"
+  - instancesout.instances|length > 0
 
 - name: slurp down the bootstrap.kubeconfig
   slurp:
@@ -42,29 +44,7 @@
     l_launch_config_security_groups: "{{ openshift_aws_launch_config_security_groups_extra }}"
     l_aws_ami_map: "{{ openshift_aws_ami_map_extra }}"
 
-
+# instances aren't scaling fast enough here, we need to wait for them
 - when: openshift_aws_wait_for_ssh | bool
-  block:
-  - name: pause and allow for instances to scale before we query them
-    pause:
-      seconds: 10
-
-  - name: fetch newly created instances
-    ec2_remote_facts:
-      region: "{{ openshift_aws_region }}"
-      filters:
-        "tag:clusterid": "{{ openshift_aws_clusterid }}"
-        "tag:host-type": node
-        instance-state-name: running
-    register: instancesout
-    retries: 20
-    delay: 3
-    until: instancesout.instances|length > 0
-
-  - name: wait for ssh to become available
-    wait_for:
-      port: 22
-      host: "{{ item.public_ip_address }}"
-      timeout: 300
-      search_regex: OpenSSH
-    with_items: "{{ instancesout.instances }}"
+  name: wait for our new nodes to come up
+  include: wait_for_groups.yml
diff --git a/roles/openshift_aws/tasks/remove_scale_group.yml b/roles/openshift_aws/tasks/remove_scale_group.yml
new file mode 100644
index 000000000..55d1af2b5
--- /dev/null
+++ b/roles/openshift_aws/tasks/remove_scale_group.yml
@@ -0,0 +1,27 @@
+---
+- name: fetch the scale groups
+  ec2_asg_facts:
+    region: "{{ openshift_aws_region }}"
+    tags:
+      "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid,
+           'version': openshift_aws_current_version} }}"
+  register: qasg
+
+- name: remove non-master scale groups
+  ec2_asg:
+    region: "{{ openshift_aws_region }}"
+    state: absent
+    name: "{{ item.auto_scaling_group_name }}"
+  when: "'master'  not in item.auto_scaling_group_name"
+  register: asg_results
+  with_items: "{{ qasg.results }}"
+  async: 600
+  poll: 0
+
+- name: join the asynch scale group removals
+  async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: jobs_results
+  with_items: "{{ asg_results.results }}"
+  until: jobs_results.finished
+  retries: 200
diff --git a/roles/openshift_aws/tasks/scale_group.yml b/roles/openshift_aws/tasks/scale_group.yml
index 097859af2..30df7545d 100644
--- a/roles/openshift_aws/tasks/scale_group.yml
+++ b/roles/openshift_aws/tasks/scale_group.yml
@@ -1,16 +1,8 @@
 ---
-- name: fetch the subnet to use in scale group
-  ec2_vpc_subnet_facts:
-    region: "{{ openshift_aws_region }}"
-    filters:
-      "tag:Name": "{{ openshift_aws_subnet_name }}"
-      vpc-id: "{{ vpcout.vpcs[0].id }}"
-  register: subnetout
-
 - name: Create the scale group
   ec2_asg:
     name: "{{ openshift_aws_scale_group_basename }} {{ item.key }}"
-    launch_config_name: "{{ openshift_aws_launch_config_basename }}-{{ item.key }}-{{ l_epoch_time }}"
+    launch_config_name: "{{ openshift_aws_launch_config_basename }}-{{ item.key }}{{ '-' ~ openshift_aws_new_version if openshift_aws_new_version != '' else '' }}"
     health_check_period: "{{ item.value.health_check.period }}"
     health_check_type: "{{ item.value.health_check.type }}"
     min_size: "{{ item.value.min_size }}"
diff --git a/roles/openshift_aws/tasks/setup_master_group.yml b/roles/openshift_aws/tasks/setup_master_group.yml
index 166f3b938..05b68f460 100644
--- a/roles/openshift_aws/tasks/setup_master_group.yml
+++ b/roles/openshift_aws/tasks/setup_master_group.yml
@@ -21,7 +21,7 @@
 
 - name: add new master to masters group
   add_host:
-    groups: "masters,etcd,nodes"
+    groups: "{{ openshift_aws_masters_groups }}"
     name: "{{ item.public_dns_name }}"
     hostname: "{{ openshift_aws_clusterid }}-master-{{ item.id[:-5] }}"
   with_items: "{{ instancesout.instances }}"
diff --git a/roles/openshift_aws/tasks/setup_scale_group_facts.yml b/roles/openshift_aws/tasks/setup_scale_group_facts.yml
new file mode 100644
index 000000000..d65fdc2de
--- /dev/null
+++ b/roles/openshift_aws/tasks/setup_scale_group_facts.yml
@@ -0,0 +1,37 @@
+---
+- name: group scale group nodes
+  ec2_remote_facts:
+    region: "{{ openshift_aws_region }}"
+    filters:
+      "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid }}}"
+  register: qinstances
+
+- name: Build new node group
+  add_host:
+    groups: oo_sg_new_nodes
+    ansible_ssh_host: "{{ item.public_dns_name }}"
+    name: "{{ item.public_dns_name }}"
+    hostname: "{{ item.public_dns_name }}"
+  when:
+  - (item.tags.version | default(False)) == openshift_aws_new_version
+  - "'node' in item.tags['host-type']"
+  with_items: "{{ qinstances.instances }}"
+
+- name: Build current node group
+  add_host:
+    groups: oo_sg_current_nodes
+    ansible_ssh_host: "{{ item.public_dns_name }}"
+    name: "{{ item.public_dns_name }}"
+    hostname: "{{ item.public_dns_name }}"
+  when:
+  - (item.tags.version | default('')) == openshift_aws_current_version
+  - "'node' in item.tags['host-type']"
+  with_items: "{{ qinstances.instances }}"
+
+- name: place all nodes into nodes group
+  add_host:
+    groups: nodes
+    ansible_ssh_host: "{{ item.public_dns_name }}"
+    name: "{{ item.public_dns_name }}"
+    hostname: "{{ item.public_dns_name }}"
+  with_items: "{{ qinstances.instances }}"
diff --git a/roles/openshift_aws/tasks/upgrade_node_group.yml b/roles/openshift_aws/tasks/upgrade_node_group.yml
new file mode 100644
index 000000000..d7851d887
--- /dev/null
+++ b/roles/openshift_aws/tasks/upgrade_node_group.yml
@@ -0,0 +1,16 @@
+---
+- fail:
+    msg: 'Please ensure the current_version and new_version variables are not the same.'
+  when:
+  - openshift_aws_current_version == openshift_aws_new_version
+
+- include: provision_nodes.yml
+
+- include: accept_nodes.yml
+
+- include: setup_scale_group_facts.yml
+
+- include: setup_master_group.yml
+  vars:
+    # we do not set etcd here as its limited to 1 or 3
+    openshift_aws_masters_groups: masters,nodes
diff --git a/roles/openshift_aws/tasks/vpc_and_subnet_id.yml b/roles/openshift_aws/tasks/vpc_and_subnet_id.yml
index aaf9b300f..1b754f863 100644
--- a/roles/openshift_aws/tasks/vpc_and_subnet_id.yml
+++ b/roles/openshift_aws/tasks/vpc_and_subnet_id.yml
@@ -6,13 +6,16 @@
       'tag:Name': "{{ openshift_aws_vpc_name }}"
   register: vpcout
 
-- name: debug
+- name: debug vcpout
   debug: var=vpcout
 
 - name: fetch the default subnet id
   ec2_vpc_subnet_facts:
     region: "{{ openshift_aws_region }}"
     filters:
-      "tag:Name": "{{ openshift_aws_subnet_name }}"
+      "availability_zone": "{{ openshift_aws_subnet_az }}"
       vpc-id: "{{ vpcout.vpcs[0].id }}"
   register: subnetout
+
+- name: debug subnetout
+  debug: var=subnetout
diff --git a/roles/openshift_aws/tasks/wait_for_groups.yml b/roles/openshift_aws/tasks/wait_for_groups.yml
new file mode 100644
index 000000000..9f1a68a2a
--- /dev/null
+++ b/roles/openshift_aws/tasks/wait_for_groups.yml
@@ -0,0 +1,31 @@
+---
+# The idea here is to wait until all scale groups are at
+# their desired capacity before continuing.
+- name: fetch the scale groups
+  ec2_asg_facts:
+    region: "{{ openshift_aws_region }}"
+    tags:
+      "{{ {'kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid} }}"
+  register: qasg
+  until: qasg.results | scale_groups_match_capacity | bool
+  delay: 10
+  retries: 60
+
+- name: fetch newly created instances
+  ec2_remote_facts:
+    region: "{{ openshift_aws_region }}"
+    filters:
+      "{{ {'tag:kubernetes.io/cluster/' ~ openshift_aws_clusterid: openshift_aws_clusterid,
+           'tag:version': openshift_aws_new_version} }}"
+  register: instancesout
+  until: instancesout.instances|length > 0
+  delay: 5
+  retries: 60
+
+- name: wait for ssh to become available
+  wait_for:
+    port: 22
+    host: "{{ item.public_ip_address }}"
+    timeout: 300
+    search_regex: OpenSSH
+  with_items: "{{ instancesout.instances }}"