Merge pull request #341 from detiber/sdodson-etcd-playbook

External clustered etcd support

18 files changed, 103 insertions, 288 deletions

diff --git a/playbooks/aws/openshift-cluster/config.yml b/playbooks/aws/openshift-cluster/config.yml
index 7188312ed..6ee539c7e 100644
--- a/playbooks/aws/openshift-cluster/config.yml
+++ b/playbooks/aws/openshift-cluster/config.yml
@@ -1,37 +1,22 @@
 ---
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
+- hosts: localhost
   gather_facts: no
   vars_files:
   - vars.yml
   tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_masters_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([])
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_config
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type_' ~ cluster_id ~ '-openshift-master'][0] }}"
-      groups: oo_first_master
-      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
-      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    when: "'tag_env-host-type_{{ cluster_id }}-openshift-master' in groups"
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
 
 - include: ../../common/openshift-cluster/config.yml
   vars:
+    g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
     openshift_cluster_id: "{{ cluster_id }}"
     openshift_debug_level: 4
     openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"
     openshift_hostname: "{{ ec2_private_ip_address }}"
     openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/launch.yml b/playbooks/aws/openshift-cluster/launch.yml
index 33e1ec25d..5db87fa90 100644
--- a/playbooks/aws/openshift-cluster/launch.yml
+++ b/playbooks/aws/openshift-cluster/launch.yml
@@ -11,6 +11,13 @@
       msg: Deployment type not supported for aws provider yet
     when: deployment_type == 'enterprise'
 
+  - include: ../../common/openshift-cluster/set_etcd_launch_facts_tasks.yml
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ etcd_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+
   - include: ../../common/openshift-cluster/set_master_launch_facts_tasks.yml
   - include: tasks/launch_instances.yml
     vars:
@@ -25,9 +32,10 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
 
-  - set_fact:
-      a_master: "{{ master_names[0] }}"
-  - add_host: name={{ a_master }} groups=service_master
+  - add_host:
+      name: "{{ master_names.0 }}"
+      groups: service_master
+    when: master_names is defined and master_names.0 is defined
 
 - include: update.yml
 
diff --git a/playbooks/aws/openshift-cluster/lookup_plugins b/playbooks/aws/openshift-cluster/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
index d643b647d..25a87aaf6 100644
--- a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
@@ -53,6 +53,15 @@
     latest_ami: "{{ ami_result.results | oo_ami_selector(ec2_image_name) }}"
     user_data: "{{ lookup('template', '../templates/user_data.j2') }}"
     volume_defs:
+      etcd:
+        root:
+          volume_size: "{{ lookup('env', 'os_etcd_root_vol_size') | default(25, true) }}"
+          device_type: "{{ lookup('env', 'os_etcd_root_vol_type') | default('gp2', true) }}"
+          iops: "{{ lookup('env', 'os_etcd_root_vol_iops') | default(500, true) }}"
+        etcd:
+          volume_size: "{{ lookup('env', 'os_etcd_vol_size') | default(32, true) }}"
+          device_type: "{{ lookup('env', 'os_etcd_vol_type') | default('gp2', true) }}"
+          iops: "{{ lookup('env', 'os_etcd_vol_iops') | default(500, true) }}"
       master:
         root:
           volume_size: "{{ lookup('env', 'os_master_root_vol_size') | default(25, true) }}"
diff --git a/playbooks/aws/openshift-cluster/templates/user_data.j2 b/playbooks/aws/openshift-cluster/templates/user_data.j2
index db14bacd1..82c2f4d57 100644
--- a/playbooks/aws/openshift-cluster/templates/user_data.j2
+++ b/playbooks/aws/openshift-cluster/templates/user_data.j2
@@ -1,4 +1,24 @@
 #cloud-config
+{% if type =='etcd' %}
+cloud_config_modules:
+- disk_setup
+- mounts
+
+mounts:
+- [ xvdb, /var/lib/etcd, xfs, "defaults" ]
+
+disk_setup:
+  xvdb:
+    table_type: mbr
+    layout: True
+
+fs_setup:
+- label: etcd_storage
+  filesystem: xfs
+  device: /dev/xvdb
+  partition: auto
+{% endif %}
+
 {% if type == 'node' %}
 mounts:
 - [ xvdb ]
diff --git a/playbooks/aws/openshift-cluster/terminate.yml b/playbooks/aws/openshift-cluster/terminate.yml
index 617d0d456..9c3703aba 100644
--- a/playbooks/aws/openshift-cluster/terminate.yml
+++ b/playbooks/aws/openshift-cluster/terminate.yml
@@ -13,4 +13,53 @@
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
     with_items: groups[scratch_group] | default([]) | difference(['localhost'])
 
-- include: ../terminate.yml
+- name: Terminate instances
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars:
+    host_vars: "{{ hostvars
+                   | oo_select_keys(groups['oo_hosts_to_terminate']) }}"
+  tasks:
+    - name: Remove tags from instances
+      ec2_tag: resource={{ item.ec2_id }} region={{ item.ec2_region }} state=absent
+      args:
+        tags:
+          env: "{{ item['ec2_tag_env'] }}"
+          host-type: "{{ item['ec2_tag_host-type'] }}"
+          env-host-type: "{{ item['ec2_tag_env-host-type'] }}"
+      with_items: host_vars
+      when: "'oo_hosts_to_terminate' in groups"
+
+    - name: Terminate instances
+      ec2:
+        state: absent
+        instance_ids: ["{{ item.ec2_id }}"]
+        region: "{{ item.ec2_region }}"
+      ignore_errors: yes
+      register: ec2_term
+      with_items: host_vars
+      when: "'oo_hosts_to_terminate' in groups"
+
+    # Fail if any of the instances failed to terminate with an error other
+    # than 403 Forbidden
+    - fail: msg=Terminating instance {{ item.ec2_id }} failed with message {{ item.msg }}
+      when: "'oo_hosts_to_terminate' in groups and item.failed and not item.msg | search(\"error: EC2ResponseError: 403 Forbidden\")"
+      with_items: ec2_term.results
+
+    - name: Stop instance if termination failed
+      ec2:
+        state: stopped
+        instance_ids: ["{{ item.item.ec2_id }}"]
+        region: "{{ item.item.ec2_region }}"
+      register: ec2_stop
+      when: "'oo_hosts_to_terminate' in groups and item.failed"
+      with_items: ec2_term.results
+
+    - name: Rename stopped instances
+      ec2_tag: resource={{ item.item.item.ec2_id }} region={{ item.item.item.ec2_region }} state=present
+      args:
+        tags:
+          Name: "{{ item.item.item.ec2_tag_Name }}-terminate"
+      with_items: ec2_stop.results
+      when: "'oo_hosts_to_terminate' in groups"
diff --git a/playbooks/aws/openshift-cluster/update.yml b/playbooks/aws/openshift-cluster/update.yml
index 5e7ab4e58..e006aa74a 100644
--- a/playbooks/aws/openshift-cluster/update.yml
+++ b/playbooks/aws/openshift-cluster/update.yml
@@ -11,7 +11,9 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"]) | default([])
+    with_items: (groups["tag_env-host-type_{{ cluster_id }}-openshift-master"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-node"] | default([]))
+                | union(groups["tag_env-host-type_{{ cluster_id }}-openshift-etcd"] | default([]))
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 
diff --git a/playbooks/aws/openshift-master/config.yml b/playbooks/aws/openshift-master/config.yml
deleted file mode 100644
index 37ab4fbe6..000000000
--- a/playbooks/aws/openshift-master/config.yml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-- name: Populate oo_masters_to_config host group
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - name: Evaluate oo_masters_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_masters_to_config
-      ansible_ssh_user: root
-    with_items: oo_host_group_exp | default([])
-
-- include: ../../common/openshift-master/config.yml
-  vars:
-    openshift_cluster_id: "{{ cluster_id }}"
-    openshift_debug_level: 4
-    openshift_deployment_type: "{{ deployment_type }}"
-    openshift_hostname: "{{ ec2_private_ip_address }}"
-    openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-master/filter_plugins b/playbooks/aws/openshift-master/filter_plugins
deleted file mode 120000
index 99a95e4ca..000000000
--- a/playbooks/aws/openshift-master/filter_plugins
+++ /dev/null
@@ -1 +0,0 @@
-../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/aws/openshift-master/launch.yml b/playbooks/aws/openshift-master/launch.yml
deleted file mode 100644
index 1cefad492..000000000
--- a/playbooks/aws/openshift-master/launch.yml
+++ /dev/null
@@ -1,70 +0,0 @@
----
-- name: Launch instance(s)
-  hosts: localhost
-  connection: local
-  gather_facts: no
-
-# TODO: modify g_ami based on deployment_type
-  vars:
-    inst_region: us-east-1
-    g_ami: ami-86781fee
-    user_data_file: user_data.txt
-
-  tasks:
-    - name: Launch instances
-      ec2:
-        state: present
-        region: "{{ inst_region }}"
-        keypair: libra
-        group: ['public']
-        instance_type: m3.large
-        image: "{{ g_ami }}"
-        count: "{{ oo_new_inst_names | length }}"
-        user_data: "{{ lookup('file', user_data_file) }}"
-        wait: yes
-      register: ec2
-
-    - name: Add new instances public IPs to the host group
-      add_host: "hostname={{ item.public_ip }} groupname=new_ec2_instances"
-      with_items: ec2.instances
-
-    - name: Add Name and environment tags to instances
-      ec2_tag: "resource={{ item.1.id }} region={{ inst_region }} state=present"
-      with_together:
-        - oo_new_inst_names
-        - ec2.instances
-      args:
-        tags:
-          Name: "{{ item.0 }}"
-
-    - name: Add other tags to instances
-      ec2_tag: resource={{ item.id }} region={{ inst_region }} state=present
-      with_items: ec2.instances
-      args:
-        tags: "{{ oo_new_inst_tags }}"
-
-    - name: Add new instances public IPs to oo_masters_to_config
-      add_host:
-        hostname: "{{ item.0 }}"
-        ansible_ssh_host: "{{ item.1.dns_name }}"
-        groupname: oo_masters_to_config
-        ec2_private_ip_address: "{{ item.1.private_ip }}"
-        ec2_ip_address: "{{ item.1.public_ip }}"
-      with_together:
-        - oo_new_inst_names
-        - ec2.instances
-
-    - name: Wait for ssh
-      wait_for: port=22 host={{ item.dns_name }}
-      with_items: ec2.instances
-
-    - name: Wait for root user setup
-      command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null root@{{ item.dns_name }} echo root user is setup"
-      register: result
-      until: result.rc == 0
-      retries: 20
-      delay: 10
-      with_items: ec2.instances
-
-# Apply the configs, seprate so that just the configs can be run by themselves
-- include: config.yml
diff --git a/playbooks/aws/openshift-master/roles b/playbooks/aws/openshift-master/roles
deleted file mode 120000
index 20c4c58cf..000000000
--- a/playbooks/aws/openshift-master/roles
+++ /dev/null
@@ -1 +0,0 @@
-../../../roles
\ No newline at end of file
diff --git a/playbooks/aws/openshift-master/terminate.yml b/playbooks/aws/openshift-master/terminate.yml
deleted file mode 100644
index 07d9961bc..000000000
--- a/playbooks/aws/openshift-master/terminate.yml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-- include: ../terminate.yml
diff --git a/playbooks/aws/openshift-node/config.yml b/playbooks/aws/openshift-node/config.yml
deleted file mode 100644
index a993a1e99..000000000
--- a/playbooks/aws/openshift-node/config.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-- name: Populate oo_nodes_to_config and oo_first_master host groups
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - name: Evaluate oo_nodes_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_nodes_to_config
-      ansible_ssh_user: root
-    with_items: oo_host_group_exp | default([])
-  - name: Evaluate oo_first_master
-    add_host:
-      name: "{{ groups['tag_env-host-type_' ~ cluster_id ~ '-openshift-master'][0] }}"
-      groups: oo_first_master
-      ansible_ssh_user: root
-
-
-- include: ../../common/openshift-node/config.yml
-  vars:
-    openshift_cluster_id: "{{ cluster_id }}"
-    openshift_debug_level: 4
-    openshift_deployment_type: "{{ deployment_type }}"
-    openshift_first_master: "{{ groups.oo_first_master.0 }}"
-    openshift_hostname: "{{ ec2_private_ip_address }}"
-    openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-node/filter_plugins b/playbooks/aws/openshift-node/filter_plugins
deleted file mode 120000
index 99a95e4ca..000000000
--- a/playbooks/aws/openshift-node/filter_plugins
+++ /dev/null
@@ -1 +0,0 @@
-../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/aws/openshift-node/launch.yml b/playbooks/aws/openshift-node/launch.yml
deleted file mode 100644
index e7d1f7310..000000000
--- a/playbooks/aws/openshift-node/launch.yml
+++ /dev/null
@@ -1,72 +0,0 @@
----
-- name: Launch instance(s)
-  hosts: localhost
-  connection: local
-  gather_facts: no
-
-# TODO: modify g_ami based on deployment_type
-  vars:
-    inst_region: us-east-1
-    g_ami: ami-86781fee
-    user_data_file: user_data.txt
-
-  tasks:
-    - name: Launch instances
-      ec2:
-        state: present
-        region: "{{ inst_region }}"
-        keypair: libra
-        group: ['public']
-        instance_type: m3.large
-        image: "{{ g_ami }}"
-        count: "{{ oo_new_inst_names | length }}"
-        user_data: "{{ lookup('file', user_data_file) }}"
-        wait: yes
-      register: ec2
-
-    - name: Add new instances public IPs to the host group
-      add_host:
-        hostname: "{{ item.public_ip }}"
-        groupname: new_ec2_instances"
-      with_items: ec2.instances
-
-    - name: Add Name and environment tags to instances
-      ec2_tag: resource={{ item.1.id }} region={{ inst_region }} state=present
-      with_together:
-        - oo_new_inst_names
-        - ec2.instances
-      args:
-        tags:
-          Name: "{{ item.0 }}"
-
-    - name: Add other tags to instances
-      ec2_tag: resource={{ item.id }} region={{ inst_region }} state=present
-      with_items: ec2.instances
-      args:
-        tags: "{{ oo_new_inst_tags }}"
-
-    - name: Add new instances public IPs to oo_nodes_to_config
-      add_host:
-        hostname: "{{ item.0 }}"
-        ansible_ssh_host: "{{ item.1.dns_name }}"
-        groupname: oo_nodes_to_config
-        ec2_private_ip_address: "{{ item.1.private_ip }}"
-        ec2_ip_address: "{{ item.1.public_ip }}"
-      with_together:
-        - oo_new_inst_names
-        - ec2.instances
-
-    - name: Wait for ssh
-      wait_for: port=22 host={{ item.dns_name }}
-      with_items: ec2.instances
-
-    - name: Wait for root user setup
-      command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null root@{{ item.dns_name }} echo root user is setup"
-      register: result
-      until: result.rc == 0
-      retries: 20
-      delay: 10
-      with_items: ec2.instances
-
-# Apply the configs, seprate so that just the configs can be run by themselves
-- include: config.yml
diff --git a/playbooks/aws/openshift-node/roles b/playbooks/aws/openshift-node/roles
deleted file mode 120000
index 20c4c58cf..000000000
--- a/playbooks/aws/openshift-node/roles
+++ /dev/null
@@ -1 +0,0 @@
-../../../roles
\ No newline at end of file
diff --git a/playbooks/aws/openshift-node/terminate.yml b/playbooks/aws/openshift-node/terminate.yml
deleted file mode 100644
index 07d9961bc..000000000
--- a/playbooks/aws/openshift-node/terminate.yml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-- include: ../terminate.yml
diff --git a/playbooks/aws/terminate.yml b/playbooks/aws/terminate.yml
deleted file mode 100644
index e9767b260..000000000
--- a/playbooks/aws/terminate.yml
+++ /dev/null
@@ -1,64 +0,0 @@
----
-- name: Populate oo_hosts_to_terminate host group
-  hosts: localhost
-  gather_facts: no
-  tasks:
-    - name: Evaluate oo_hosts_to_terminate
-      add_host: name={{ item }} groups=oo_hosts_to_terminate
-      with_items: oo_host_group_exp | default([])
-
-- name: Gather dynamic inventory variables for hosts to terminate
-  hosts: oo_hosts_to_terminate
-  gather_facts: no
-
-- name: Terminate instances
-  hosts: localhost
-  connection: local
-  gather_facts: no
-  vars:
-    host_vars: "{{ hostvars
-        | oo_select_keys(groups['oo_hosts_to_terminate']) }}"
-  tasks:
-    - name: Remove tags from instances
-      ec2_tag: resource={{ item.ec2_id }} region={{ item.ec2_region }} state=absent
-      args:
-        tags:
-          env: "{{ item['ec2_tag_env'] }}"
-          host-type: "{{ item['ec2_tag_host-type'] }}"
-          env-host-type: "{{ item['ec2_tag_env-host-type'] }}"
-      with_items: host_vars
-      when: "'oo_hosts_to_terminate' in groups"
-
-    - name: Terminate instances
-      ec2:
-        state: absent
-        instance_ids: ["{{ item.ec2_id }}"]
-        region: "{{ item.ec2_region }}"
-      ignore_errors: yes
-      register: ec2_term
-      with_items: host_vars
-      when: "'oo_hosts_to_terminate' in groups"
-
-    # Fail if any of the instances failed to terminate with an error other
-    # than 403 Forbidden
-    - fail: msg=Terminating instance {{ item.item.ec2_id }} failed with message {{ item.msg }}
-      when: "'oo_hosts_to_terminate' in groups and item.failed and not item.msg | search(\"error: EC2ResponseError: 403 Forbidden\")"
-      with_items: ec2_term.results
-
-    - name: Stop instance if termination failed
-      ec2:
-        state: stopped
-        instance_ids: ["{{ item.item.ec2_id }}"]
-        region: "{{ item.item.ec2_region }}"
-      register: ec2_stop
-      when: item.failed
-      with_items: ec2_term.results
-      when: "'oo_hosts_to_terminate' in groups"
-
-    - name: Rename stopped instances
-      ec2_tag: resource={{ item.item.item.ec2_id }} region={{ item.item.item.ec2_region }} state=present
-      args:
-        tags:
-          Name: "{{ item.item.item.ec2_tag_Name }}-terminate"
-      with_items: ec2_stop.results
-      when: "'oo_hosts_to_terminate' in groups"