From 2a7131b9403a4b22ebc55606814f604f723dc826 Mon Sep 17 00:00:00 2001
From: Sylvain Baubeau <sbaubeau@redhat.com>
Date: Tue, 13 Oct 2015 16:36:01 +0200
Subject: Add flannel support

Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
---
 playbooks/common/openshift-master/config.yml | 8 ++++++++
 playbooks/common/openshift-node/config.yml   | 3 +++
 2 files changed, 11 insertions(+)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 1dec923fc..54d61da06 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -247,6 +247,14 @@
     when: ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
       (osm_use_cockpit | bool or osm_use_cockpit is undefined )
 
+- name: Configure flannel
+  hosts: oo_first_master
+  vars:
+    etcd_urls: "{{ openshift.master.etcd_urls }}"
+  roles:
+  - role: flannel_register
+    when: openshift.common.use_flannel | bool
+
 # Additional instance config for online deployments
 - name: Additional instance config
   hosts: oo_masters_deployment_type_online
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index a14ca8e11..e39e9164c 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -84,6 +84,7 @@
   vars:
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+    etcd_urls: "{{ hostvars[groups.oo_first_master.0].openshift.master.etcd_urls }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -100,6 +101,8 @@
     when: certs_missing
   roles:
   - openshift_node
+  - role: flannel
+    when: openshift.common.use_flannel | bool
   - role: nickhammond.logrotate
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool
-- 
cgit v1.2.3


From bb30f53935399fee9dcaf42664fe8678dd157ee1 Mon Sep 17 00:00:00 2001
From: Sylvain Baubeau <sbaubeau@redhat.com>
Date: Mon, 19 Oct 2015 15:50:07 +0200
Subject: Generate etcd certificats for flannel when is not embedded

---
 playbooks/common/openshift-node/config.yml | 70 ++++++++++++++++++++++++++++++
 roles/flannel/defaults/main.yaml           |  6 +--
 roles/flannel_register/defaults/main.yaml  |  2 +-
 3 files changed, 74 insertions(+), 4 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index e39e9164c..5f0f329c4 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -38,6 +38,21 @@
       node_subdir: node-{{ openshift.common.hostname }}
       config_dir: "{{ openshift.common.config_base }}/generated-configs/node-{{ openshift.common.hostname }}"
       node_cert_dir: "{{ openshift.common.config_base }}/node"
+  - name: Check status of flannel external etcd certificates
+    stat:
+      path: "{{ openshift.common.config_base }}/node/{{ item }}"
+    with_items:
+    - node.etcd-client.crt
+    - node.etcd-ca.crt
+    register: g_external_etcd_flannel_cert_stat_result
+  - set_fact:
+      etcd_client_flannel_certs_missing: "{{ g_external_etcd_flannel_cert_stat_result.results
+                                             | map(attribute='stat.exists')
+                                             | list | intersect([false])}}"
+      etcd_cert_subdir: openshift-node-{{ openshift.common.hostname }}
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/node"
+      etcd_cert_prefix: node.etcd-
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
 
 - name: Create temp directory for syncing certs
   hosts: localhost
@@ -50,6 +65,60 @@
     register: mktemp
     changed_when: False
 
+- name: Configure flannel etcd certificates
+  hosts: oo_first_etcd
+  vars:
+    etcd_generated_certs_dir: /etc/etcd/generated_certs
+    etcd_needing_client_certs: "{{ hostvars
+                                   | oo_select_keys(groups['oo_nodes_to_config'])
+                                   | oo_filter_list(filter_attr='etcd_client_flannel_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
+  pre_tasks:
+  roles:
+  - role: etcd_certificates
+  post_tasks:
+  - name: Create a tarball of the etcd flannel certs
+    command: >
+      tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz
+        -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} .
+    args:
+      creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+    with_items: etcd_needing_client_certs
+  - name: Retrieve the etcd cert tarballs
+    fetch:
+      src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_needing_client_certs
+
+- name: Copy the external etcd flannel certs to the nodes
+  hosts: oo_nodes_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
+  tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ openshift.common.config_base }}/node"
+      state: directory
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz"
+      dest: "{{ etcd_cert_config_dir }}"
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+  - file:
+      path: "{{ etcd_cert_config_dir }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+    - node.etcd-client.crt
+    - node.etcd-client.key
+    - node.etcd-ca.crt
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
+
 - name: Create node certificates
   hosts: oo_first_master
   vars:
@@ -85,6 +154,7 @@
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
     etcd_urls: "{{ hostvars[groups.oo_first_master.0].openshift.master.etcd_urls }}"
+    embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
diff --git a/roles/flannel/defaults/main.yaml b/roles/flannel/defaults/main.yaml
index a9acab634..34cebda9c 100644
--- a/roles/flannel/defaults/main.yaml
+++ b/roles/flannel/defaults/main.yaml
@@ -3,6 +3,6 @@ flannel_interface: "{{ ansible_default_ipv4.interface }}"
 flannel_etcd_key: /openshift.com/network
 etcd_hosts: "{{ etcd_urls }}"
 etcd_conf_dir: "{{ openshift.common.config_base }}/node"
-etcd_peer_ca_file: "{{ etcd_conf_dir }}/ca.crt"
-etcd_peer_cert_file: "{{ etcd_conf_dir }}/system:node:{{ openshift.common.hostname }}.crt"
-etcd_peer_key_file: "{{ etcd_conf_dir }}/system:node:{{ openshift.common.hostname }}.key"
+etcd_peer_ca_file: "{{ etcd_conf_dir }}/{{ 'ca' if (embedded_etcd | bool) else 'node.etcd-ca' }}.crt"
+etcd_peer_cert_file: "{{ etcd_conf_dir }}/{{ 'system:node:' + openshift.common.hostname if (embedded_etcd | bool) else 'node.etcd-client' }}.crt"
+etcd_peer_key_file: "{{ etcd_conf_dir }}/{{ 'system:node:' + openshift.common.hostname if (embedded_etcd | bool) else 'node.etcd-client' }}.key"
diff --git a/roles/flannel_register/defaults/main.yaml b/roles/flannel_register/defaults/main.yaml
index e4b06b49b..7a454d1d2 100644
--- a/roles/flannel_register/defaults/main.yaml
+++ b/roles/flannel_register/defaults/main.yaml
@@ -5,7 +5,7 @@ flannel_subnet_len: "{{ subnet_len | default(24, true) }}"
 flannel_etcd_key: /openshift.com/network
 etcd_hosts: "{{ etcd_urls }}"
 etcd_conf_dir: "{{ openshift.common.config_base }}/master"
-etcd_peer_ca_file: "{{ etcd_conf_dir }}/ca.crt"
+etcd_peer_ca_file: "{{ etcd_conf_dir + '/ca.crt' if (openshift.master.embedded_etcd | bool) else etcd_conf_dir + '/master.etcd-ca.crt' }}"
 etcd_peer_cert_file: "{{ etcd_conf_dir }}/master.etcd-client.crt"
 etcd_peer_key_file: "{{ etcd_conf_dir }}/master.etcd-client.key"
 
-- 
cgit v1.2.3


From 4db5a237af8b826b52622e1b79017df98611ab10 Mon Sep 17 00:00:00 2001
From: Sylvain Baubeau <sbaubeau@redhat.com>
Date: Tue, 27 Oct 2015 16:16:44 +0100
Subject: Check etcd certs exist for flannel when its support is enabled

---
 playbooks/common/openshift-node/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 5f0f329c4..ba96b4a78 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -52,7 +52,7 @@
       etcd_cert_subdir: openshift-node-{{ openshift.common.hostname }}
       etcd_cert_config_dir: "{{ openshift.common.config_base }}/node"
       etcd_cert_prefix: node.etcd-
-    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config and (openshift.common.use_flannel | bool)
 
 - name: Create temp directory for syncing certs
   hosts: localhost
-- 
cgit v1.2.3


From 3778662ef816b2bb0a3788ed65229b45622a0139 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Fri, 21 Aug 2015 23:49:49 -0400
Subject: Start of true master ha

---
 filter_plugins/oo_filters.py                 | 16 ++++++++++
 playbooks/common/openshift-master/config.yml | 26 +++++++++++++++
 roles/haproxy/README.md                      | 34 ++++++++++++++++++++
 roles/haproxy/defaults/main.yml              | 13 ++++++++
 roles/haproxy/handlers/main.yml              |  5 +++
 roles/haproxy/meta/main.yml                  | 12 +++++++
 roles/haproxy/tasks/main.yml                 | 25 +++++++++++++++
 roles/haproxy/templates/haproxy.cfg.j2       | 48 ++++++++++++++++++++++++++++
 roles/openshift_master/tasks/main.yml        |  8 ++---
 9 files changed, 183 insertions(+), 4 deletions(-)
 create mode 100644 roles/haproxy/README.md
 create mode 100644 roles/haproxy/defaults/main.yml
 create mode 100644 roles/haproxy/handlers/main.yml
 create mode 100644 roles/haproxy/meta/main.yml
 create mode 100644 roles/haproxy/tasks/main.yml
 create mode 100644 roles/haproxy/templates/haproxy.cfg.j2

(limited to 'playbooks/common')

diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index dfd9a111e..80bce80f0 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -242,6 +242,21 @@ class FilterModule(object):
         '''
         return string.split(separator)
 
+    @staticmethod
+    def oo_haproxy_backend_masters(hosts):
+        ''' This takes an array of dicts and returns an array of dicts
+            to be used as a backend for the haproxy role
+        '''
+        servers = []
+        for idx, host_info in enumerate(hosts):
+            server = dict(name="master%s" % idx)
+            server_ip = host_info['openshift']['common']['ip']
+            server_port = host_info['openshift']['master']['api_port']
+            server['address'] = "%s:%s" % (server_ip, server_port)
+            server['opts'] = 'check ssl verify none'
+            servers.append(server)
+        return servers
+
     @staticmethod
     def oo_filter_list(data, filter_attr=None):
         ''' This returns a list, which contains all items where filter_attr
@@ -408,4 +423,5 @@ class FilterModule(object):
             "oo_filter_list": self.oo_filter_list,
             "oo_parse_heat_stack_outputs": self.oo_parse_heat_stack_outputs,
             "oo_parse_certificate_names": self.oo_parse_certificate_names
+            "oo_haproxy_backend_masters": self.oo_haproxy_backend_masters
         }
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 59c4b2370..84ce65f48 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -34,7 +34,9 @@
       - role: common
         local_facts:
           hostname: "{{ openshift_hostname | default(None) }}"
+          ip: "{{ openshift_ip | default(None) }}"
           public_hostname: "{{ openshift_public_hostname | default(None) }}"
+          public_ip: "{{ openshift_public_ip | default(None) }}"
           deployment_type: "{{ openshift_deployment_type }}"
       - role: master
         local_facts:
@@ -207,6 +209,30 @@
       parsed_named_certificates: "{{ openshift_master_named_certificates | oo_parse_certificate_names(master_cert_config_dir, openshift.common.internal_hostnames) }}"
     when: openshift_master_named_certificates is defined
 
+- name: Compute haproxy_backend_servers
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - set_fact:
+      haproxy_backend_servers: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_haproxy_backend_masters }}"
+
+- name: Configure load balancers
+  hosts: oo_first_master
+  vars:
+    haproxy_frontends:
+    - name: atomic-openshift
+      bind: "*:80"
+      default_backend: atomic-openshift
+    haproxy_backends:
+    - name: atomic-openshift
+      balance: roundrobin
+      servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
+  roles:
+  - role: haproxy
+    when: groups.oo_masters_to_config | length > 1
+
 - name: Configure master instances
   hosts: oo_masters_to_config
   vars:
diff --git a/roles/haproxy/README.md b/roles/haproxy/README.md
new file mode 100644
index 000000000..5bc415066
--- /dev/null
+++ b/roles/haproxy/README.md
@@ -0,0 +1,34 @@
+HAProxy
+=======
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/haproxy/defaults/main.yml b/roles/haproxy/defaults/main.yml
new file mode 100644
index 000000000..c002efdbc
--- /dev/null
+++ b/roles/haproxy/defaults/main.yml
@@ -0,0 +1,13 @@
+---
+haproxy_frontends:
+- name: main
+  bind: "*:80"
+  default_backend: default
+
+haproxy_backends:
+- name: default
+  balance: roundrobin
+  servers:
+  - name: web01
+    address: 127.0.0.1:9000
+    opts: check
diff --git a/roles/haproxy/handlers/main.yml b/roles/haproxy/handlers/main.yml
new file mode 100644
index 000000000..ee60adcab
--- /dev/null
+++ b/roles/haproxy/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: restart haproxy
+  service:
+    name: haproxy
+    state: restarted
diff --git a/roles/haproxy/meta/main.yml b/roles/haproxy/meta/main.yml
new file mode 100644
index 000000000..e02d8f53c
--- /dev/null
+++ b/roles/haproxy/meta/main.yml
@@ -0,0 +1,12 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description: HAProxy
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+dependencies: []
diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml
new file mode 100644
index 000000000..5638b7313
--- /dev/null
+++ b/roles/haproxy/tasks/main.yml
@@ -0,0 +1,25 @@
+---
+- name: Install haproxy
+  yum:
+    pkg: haproxy
+    state: present
+
+- name: Configure haproxy
+  template:
+    src: haproxy.cfg.j2
+    dest: /etc/haproxy/haproxy.cfg
+    owner: root
+    group: root
+    mode: 0644
+  notify: restart haproxy
+
+- name: Enable and start haproxy
+  service:
+    name: haproxy
+    state: started
+    enabled: yes
+  register: start_result
+
+- name: Pause 30 seconds if haproxy was just started
+  pause: seconds=30
+  when: start_result | changed
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
new file mode 100644
index 000000000..bfcdcfdb1
--- /dev/null
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -0,0 +1,48 @@
+# Global settings
+#---------------------------------------------------------------------
+global
+    chroot      /var/lib/haproxy
+    pidfile     /var/run/haproxy.pid
+    maxconn     4000
+    user        haproxy
+    group       haproxy
+    daemon
+
+    # turn on stats unix socket
+    stats socket /var/lib/haproxy/stats
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 3
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           10s
+    maxconn                 3000
+
+{% for frontend in haproxy_frontends %}
+frontend  {{ frontend.name }}
+    bind {{ frontend.bind }}
+    default_backend {{ frontend.default_backend }}
+{% endfor %}
+
+{% for backend in haproxy_backends %}
+backend {{ backend.name }}
+    balance {{ backend.balance }}
+{% for server in backend.servers %}
+    server      {{ server.name }} {{ server.address }} {{ server.opts }}
+{% endfor %}
+{% endfor %}
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index e1049abdd..abe652bcb 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -8,9 +8,9 @@
     - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods
   when: openshift_master_oauth_grant_method is defined
 
-- fail:
-    msg: "openshift_master_cluster_password must be set for multi-master installations"
-  when: openshift_master_ha | bool and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
+#- fail:
+#    msg: "openshift_master_cluster_password must be set for multi-master installations"
+#  when: openshift_master_ha | bool and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
 
 - name: Set master facts
   openshift_facts:
@@ -144,7 +144,7 @@
 
 - name: Start and enable master
   service: name={{ openshift.common.service_type }}-master enabled=yes state=started
-  when: not openshift_master_ha | bool
+#  when: not openshift_master_ha | bool
   register: start_result
 
 - set_fact:
-- 
cgit v1.2.3


From 51bcc78aea4015bf23d06b621b57de675b21e7cf Mon Sep 17 00:00:00 2001
From: Andrew Butcher <abutcher@redhat.com>
Date: Thu, 22 Oct 2015 13:58:38 -0400
Subject: additional native ha changes

---
 playbooks/common/openshift-master/config.yml       | 10 +++++-----
 roles/openshift_master/tasks/main.yml              |  1 +
 roles/openshift_master/templates/master.yaml.v1.j2 |  3 +--
 3 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 84ce65f48..e5357f6e3 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -239,7 +239,7 @@
     named_certificates: "{{ hostvars[groups['oo_first_master'][0]]['parsed_named_certificates'] | default([])}}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    embedded_etcd: "{{ openshift.master.embedded_etcd }}"
+    openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -264,11 +264,11 @@
 - name: Additional master configuration
   hosts: oo_first_master
   vars:
-    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
+  #openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+  #  omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
   roles:
-  - role: openshift_master_cluster
-    when: openshift_master_ha | bool
+#  - role: openshift_master_cluster
+#    when: openshift_master_ha | bool
   - openshift_examples
   - role: openshift_cluster_metrics
     when: openshift.common.use_cluster_metrics | bool
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index abe652bcb..f11582ce7 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -63,6 +63,7 @@
       controller_args: "{{ osm_controller_args | default(None) }}"
       infra_nodes: "{{ num_infra | default(None) }}"
       disabled_features: "{{ osm_disabled_features | default(None) }}"
+      master_count: "{{ openshift_master_count | default(None) }}"
 
 - name: Install Master package
   yum: pkg={{ openshift.common.service_type }}-master state=present
diff --git a/roles/openshift_master/templates/master.yaml.v1.j2 b/roles/openshift_master/templates/master.yaml.v1.j2
index 9547a6945..877c44772 100644
--- a/roles/openshift_master/templates/master.yaml.v1.j2
+++ b/roles/openshift_master/templates/master.yaml.v1.j2
@@ -80,8 +80,7 @@ kubernetesMasterConfig:
   - v1
   apiServerArguments: {{ api_server_args if api_server_args is defined else 'null' }}
   controllerArguments: {{ controller_args if controller_args is defined else 'null' }}
-{# TODO: support overriding masterCount #}
-  masterCount: 1
+  masterCount: {{ openshift.master.master_count }}
   masterIP: ""
   podEvictionTimeout: ""
   proxyClientInfo:
-- 
cgit v1.2.3


From 18c877db73dcb63b1402322fe8352505006e4985 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Tue, 25 Aug 2015 08:42:20 -0400
Subject: additional ha related updates

---
 playbooks/byo/openshift-cluster/config.yml         |  1 +
 playbooks/common/openshift-cluster/config.yml      |  8 +++
 playbooks/common/openshift-master/config.yml       | 49 +++++++++++--
 roles/haproxy/defaults/main.yml                    |  3 +-
 roles/haproxy/templates/haproxy.cfg.j2             | 25 ++++++-
 roles/openshift_master/handlers/main.yml           |  8 +++
 roles/openshift_master/tasks/main.yml              | 84 +++++++++++++++++++++-
 roles/openshift_master/templates/master.yaml.v1.j2 | 18 +++--
 8 files changed, 182 insertions(+), 14 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/byo/openshift-cluster/config.yml b/playbooks/byo/openshift-cluster/config.yml
index 9e50a4a18..411c7e660 100644
--- a/playbooks/byo/openshift-cluster/config.yml
+++ b/playbooks/byo/openshift-cluster/config.yml
@@ -4,6 +4,7 @@
     g_etcd_group: "{{ 'etcd' }}"
     g_masters_group: "{{ 'masters' }}"
     g_nodes_group: "{{ 'nodes' }}"
+    g_lb_group: "{{ 'lb' }}"
     openshift_cluster_id: "{{ cluster_id | default('default') }}"
     openshift_debug_level: 2
     openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index 57de7130b..b66ca4709 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -1,6 +1,14 @@
 ---
 - include: evaluate_groups.yml
 
+  - name: Evaluate oo_lb_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_lb_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_lb_group] | default(groups[g_masters_group]) | default([])
+
 - include: ../openshift-etcd/config.yml
 
 - include: ../openshift-master/config.yml
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index e5357f6e3..e223e3d57 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -209,7 +209,24 @@
       parsed_named_certificates: "{{ openshift_master_named_certificates | oo_parse_certificate_names(master_cert_config_dir, openshift.common.internal_hostnames) }}"
     when: openshift_master_named_certificates is defined
 
-- name: Compute haproxy_backend_servers
+- name: Fetch master server certificate for load balancer
+  hosts: oo_first_master
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  tasks:
+  - file:
+      path: "{{ sync_tmpdir }}/haproxy_cert"
+      state: directory
+  - fetch:
+      src: /etc/origin/master/master.server.crt
+      dest: "{{ sync_tmpdir }}/haproxy_cert/server.crt"
+      flat: yes
+  - fetch:
+      src: /etc/origin/master/master.server.key
+      dest: "{{ sync_tmpdir }}/haproxy_cert/server.key"
+      flat: yes
+
+- name: Compute haproxy_backend_servers and combine certificate
   hosts: localhost
   connection: local
   sudo: false
@@ -217,24 +234,44 @@
   tasks:
   - set_fact:
       haproxy_backend_servers: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_haproxy_backend_masters }}"
+  - shell: cat server.crt server.key > server.pem
+    args:
+      chdir: "{{ g_master_mktemp.stdout }}/haproxy_cert"
+      creates: "{{ g_master_mktemp.stdout }}/haproxy_cert/server.pem"
+
 
 - name: Configure load balancers
-  hosts: oo_first_master
+  hosts: oo_lb_to_config
   vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     haproxy_frontends:
-    - name: atomic-openshift
-      bind: "*:80"
-      default_backend: atomic-openshift
+    - name: atomic-openshift-api
+      options:
+      - tcplog
+      binds:
+      - "*:{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }} ssl crt /etc/haproxy/server.pem"
+      default_backend: atomic-openshift-api
     haproxy_backends:
-    - name: atomic-openshift
+    - name: atomic-openshift-api
       balance: roundrobin
       servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
+  pre_tasks:
+  - file:
+      path: /etc/haproxy
+      state: directory
+  - copy:
+      src: "{{ sync_tmpdir }}/haproxy_cert/server.pem"
+      dest: /etc/haproxy/server.pem
+      mode: 0600
+      owner: root
+      group: root
   roles:
   - role: haproxy
     when: groups.oo_masters_to_config | length > 1
 
 - name: Configure master instances
   hosts: oo_masters_to_config
+  serial: 1
   vars:
     named_certificates: "{{ hostvars[groups['oo_first_master'][0]]['parsed_named_certificates'] | default([])}}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
diff --git a/roles/haproxy/defaults/main.yml b/roles/haproxy/defaults/main.yml
index c002efdbc..16e9af4d1 100644
--- a/roles/haproxy/defaults/main.yml
+++ b/roles/haproxy/defaults/main.yml
@@ -1,7 +1,8 @@
 ---
 haproxy_frontends:
 - name: main
-  bind: "*:80"
+  binds:
+  - "*:80"
   default_backend: default
 
 haproxy_backends:
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
index bfcdcfdb1..fddf0ede1 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -35,13 +35,36 @@ defaults
 
 {% for frontend in haproxy_frontends %}
 frontend  {{ frontend.name }}
-    bind {{ frontend.bind }}
+{% for bind in frontend.binds %}
+    bind {{ bind }}
+{% endfor %}
     default_backend {{ frontend.default_backend }}
+{% if 'mode' in frontend %}
+    mode {{ frontend.mode }}
+{% endif %}
+{% if 'options' in frontend %}
+{% for option in frontend.options %}
+    option {{ option }}
+{% endfor %}
+{% endif %}
+{% if 'redirects' in frontend %}
+{% for redirect in frontend.redirects %}
+    redirect {{ redirect }}
+{% endfor %}
+{% endif %}
 {% endfor %}
 
 {% for backend in haproxy_backends %}
 backend {{ backend.name }}
     balance {{ backend.balance }}
+{% if 'mode' in backend %}
+    mode {{ backend.mode }}
+{% endif %}
+{% if 'options' in backend %}
+{% for option in backend.options %}
+    option {{ option }}
+{% endfor %}
+{% endif %}
 {% for server in backend.servers %}
     server      {{ server.name }} {{ server.address }} {{ server.opts }}
 {% endfor %}
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index 37028e0f6..9ce4f512b 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -2,3 +2,11 @@
 - name: restart master
   service: name={{ openshift.common.service_type }}-master state=restarted
   when: (not openshift_master_ha | bool) and (not master_service_status_changed | default(false))
+
+- name: restart master api
+  service: name={{ openshift.common.service_type }}-master-api state=restarted
+  when: openshift_master_ha | bool
+
+- name: restart master controllers
+  service: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  when: openshift_master_ha | bool
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index f11582ce7..b23c19d37 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -91,6 +91,8 @@
     creates: "{{ openshift_master_policy }}"
   notify:
   - restart master
+  - restart master api
+  - restart master controllers
 
 - name: Create the scheduler config
   template:
@@ -99,6 +101,8 @@
     backup: true
   notify:
   - restart master
+  - restart master api
+  - restart master controllers
 
 - name: Install httpd-tools if needed
   yum: pkg=httpd-tools state=present
@@ -121,6 +125,30 @@
   when: item.kind == 'HTPasswdPasswordIdentityProvider'
   with_items: openshift.master.identity_providers
 
+# workaround for missing systemd unit files for controllers/api
+- name: Create the api service file
+  copy:
+    src: atomic-openshift-master-api.service
+    dest: /usr/lib/systemd/system/atomic-openshift-master-api.service
+    force: no
+- name: Create the controllers service file
+  copy:
+    src: atomic-openshift-master-controllers.service
+    dest: /usr/lib/systemd/system/atomic-openshift-master-controllers.service
+    force: no
+- name: Create the api env file
+  copy:
+    src: atomic-openshift-master-api
+    dest: /etc/sysconfig/atomic-openshift-master-api
+    force: no
+- name: Create the controllers env file
+  copy:
+    src: atomic-openshift-master-controllers
+    dest: /etc/sysconfig/atomic-openshift-master-controllers
+    force: no
+- command: systemctl daemon-reload
+# end workaround for missing systemd unit files
+
 # TODO: add the validate parameter when there is a validation command to run
 - name: Create master config
   template:
@@ -129,6 +157,8 @@
     backup: true
   notify:
   - restart master
+  - restart master api
+  - restart master controllers
 
 - name: Configure master settings
   lineinfile:
@@ -143,9 +173,61 @@
   notify:
   - restart master
 
+- name: Configure master api settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+  with_items:
+    - regex: '^OPTIONS='
+      line: "OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen=https://0.0.0.0:8443 --master=https://{{ openshift.common.ip }}:8443"
+    - regex: '^CONFIG_FILE='
+      line: "CONFIG_FILE={{ openshift_master_config_file }}"
+  notify:
+  - restart master api
+
+- name: Configure master controller settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+  with_items:
+    - regex: '^OPTIONS='
+      line: "OPTIONS=--loglevel={{ openshift.master.debug_level }} --listen=https://0.0.0.0:8444"
+    - regex: '^CONFIG_FILE='
+      line: "CONFIG_FILE={{ openshift_master_config_file }}"
+  notify:
+  - restart master controllers
+
 - name: Start and enable master
   service: name={{ openshift.common.service_type }}-master enabled=yes state=started
-#  when: not openshift_master_ha | bool
+  when: not openshift_master_ha | bool
+  register: start_result
+
+# workaround for start bug when configuring ha
+- name: Start master for ha workaround
+  service: name={{ openshift.common.service_type }}-master state=started
+  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
+
+- name: pause for 30 seconds to let master finish starting up for ha workaround
+  pause: seconds=30
+  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
+
+- name: Stop master for ha workaround
+  service: name={{ openshift.common.service_type }}-master state=stopped
+  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
+# end workaround for start bug when configuring ha
+
+- fail:
+
+- name: Start and enable master api
+  service: name={{ openshift.common.service_type }}-master-api enabled=yes state=started
+  when: openshift_master_ha | bool
+  register: start_result
+
+- name: Start and enable master controller
+  service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started
+  when: openshift_master_ha | bool
   register: start_result
 
 - set_fact:
diff --git a/roles/openshift_master/templates/master.yaml.v1.j2 b/roles/openshift_master/templates/master.yaml.v1.j2
index 877c44772..3f2c51417 100644
--- a/roles/openshift_master/templates/master.yaml.v1.j2
+++ b/roles/openshift_master/templates/master.yaml.v1.j2
@@ -10,13 +10,16 @@ assetConfig:
   publicURL: {{ openshift.master.public_console_url }}/
   servingInfo:
     bindAddress: {{ openshift.master.bind_addr }}:{{ openshift.master.console_port }}
+    bindNetwork: tcp4
     certFile: master.server.crt
     clientCA: ""
     keyFile: master.server.key
     maxRequestsInFlight: 0
     requestTimeoutSeconds: 0
+controllerLeaseTTL: 0
+controllers: '*'
 corsAllowedOrigins:
-{% for origin in ['127.0.0.1', 'localhost', openshift.common.hostname, openshift.common.ip, openshift.common.public_hostname, openshift.common.public_ip] | unique %}
+{% for origin in ['127.0.0.1', 'localhost', openshift.common.ip, openshift.common.public_ip] | union(openshift.common.all_hostnames) | unique %}
   - {{ origin }}
 {% endfor %}
 {% for custom_origin in openshift.master.custom_cors_origins | default("") %}
@@ -29,8 +32,10 @@ corsAllowedOrigins:
 disabledFeatures: {{ openshift.master.disabled_features | to_json }}
 {% endif %}
 {% if openshift.master.embedded_dns | bool %}
+disabledFeatures: null
 dnsConfig:
   bindAddress: {{ openshift.master.bind_addr }}:{{ openshift.master.dns_port }}
+  bindNetwork: tcp4
 {% endif %}
 etcdClientInfo:
   ca: {{ "ca.crt" if (openshift.master.embedded_etcd | bool) else "master.etcd-ca.crt" }}
@@ -81,13 +86,13 @@ kubernetesMasterConfig:
   apiServerArguments: {{ api_server_args if api_server_args is defined else 'null' }}
   controllerArguments: {{ controller_args if controller_args is defined else 'null' }}
   masterCount: {{ openshift.master.master_count }}
-  masterIP: ""
-  podEvictionTimeout: ""
+  masterIP: {{ openshift.common.ip }}
+  podEvictionTimeout: 5m
   proxyClientInfo:
     certFile: master.proxy-client.crt
     keyFile: master.proxy-client.key
   schedulerConfigFile: {{ openshift_master_scheduler_conf }}
-  servicesNodePortRange: ""
+  servicesNodePortRange: 30000-32767
   servicesSubnet: {{ openshift.master.portal_net }}
   staticNodeNames: {{ openshift_node_ips | default([], true) }}
 {% endif %}
@@ -105,6 +110,7 @@ networkConfig:
 # serviceNetworkCIDR must match kubernetesMasterConfig.servicesSubnet
   serviceNetworkCIDR: {{ openshift.master.portal_net }}
 {% include 'v1_partials/oauthConfig.j2' %}
+pauseControllers: false
 policyConfig:
   bootstrapPolicyFile: {{ openshift_master_policy }}
   openshiftInfrastructureNamespace: openshift-infra
@@ -118,8 +124,9 @@ projectConfig:
     mcsLabelsPerProject: {{ openshift.master.mcs_labels_per_project }}
     uidAllocatorRange: "{{ openshift.master.uid_allocator_range  }}"
 routingConfig:
-  subdomain:  "{{ openshift.master.default_subdomain | default("") }}"
+  subdomain:  "{{ openshift.master.default_subdomain | default("router.default.svc.cluster.local") }}"
 serviceAccountConfig:
+  limitSecretReferences: false
   managedNames:
   - default
   - builder
@@ -130,6 +137,7 @@ serviceAccountConfig:
   - serviceaccounts.public.key
 servingInfo:
   bindAddress: {{ openshift.master.bind_addr }}:{{ openshift.master.api_port }}
+  bindNetwork: tcp4
   certFile: master.server.crt
   clientCA: ca.crt
   keyFile: master.server.key
-- 
cgit v1.2.3


From ac0f4cb56e1469e9033e3a218265bc70f774624d Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Tue, 25 Aug 2015 14:40:08 -0400
Subject: more tweaks

---
 filter_plugins/oo_filters.py                       |  2 +-
 playbooks/common/openshift-master/config.yml       | 44 +++++-----------------
 roles/haproxy/defaults/main.yml                    |  7 ++++
 roles/haproxy/meta/main.yml                        |  4 +-
 roles/haproxy/templates/haproxy.cfg.j2             |  9 ++++-
 .../files/atomic-openshift-master-api              |  9 +++++
 .../files/atomic-openshift-master-api.service      | 21 +++++++++++
 .../files/atomic-openshift-master-controllers      |  9 +++++
 .../atomic-openshift-master-controllers.service    | 22 +++++++++++
 roles/openshift_master/tasks/main.yml              | 21 +++--------
 roles/openshift_master_ca/tasks/main.yml           |  2 +-
 11 files changed, 94 insertions(+), 56 deletions(-)
 create mode 100644 roles/openshift_master/files/atomic-openshift-master-api
 create mode 100644 roles/openshift_master/files/atomic-openshift-master-api.service
 create mode 100644 roles/openshift_master/files/atomic-openshift-master-controllers
 create mode 100644 roles/openshift_master/files/atomic-openshift-master-controllers.service

(limited to 'playbooks/common')

diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index 80bce80f0..4e4f7507c 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -253,7 +253,7 @@ class FilterModule(object):
             server_ip = host_info['openshift']['common']['ip']
             server_port = host_info['openshift']['master']['api_port']
             server['address'] = "%s:%s" % (server_ip, server_port)
-            server['opts'] = 'check ssl verify none'
+            server['opts'] = 'check'
             servers.append(server)
         return servers
 
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index e223e3d57..67068e001 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -170,6 +170,10 @@
     masters_needing_certs: "{{ hostvars
                                | oo_select_keys(groups['oo_masters_to_config'] | difference(groups['oo_first_master']))
                                | oo_filter_list(filter_attr='master_certs_missing') }}"
+    master_hostnames: "{{ hostvars
+                               | oo_select_keys(groups['oo_masters_to_config'])
+                               | oo_collect('openshift.common.all_hostnames')
+                               | oo_flatten | unique }}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
   roles:
   - openshift_master_certificates
@@ -209,24 +213,7 @@
       parsed_named_certificates: "{{ openshift_master_named_certificates | oo_parse_certificate_names(master_cert_config_dir, openshift.common.internal_hostnames) }}"
     when: openshift_master_named_certificates is defined
 
-- name: Fetch master server certificate for load balancer
-  hosts: oo_first_master
-  vars:
-    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-  tasks:
-  - file:
-      path: "{{ sync_tmpdir }}/haproxy_cert"
-      state: directory
-  - fetch:
-      src: /etc/origin/master/master.server.crt
-      dest: "{{ sync_tmpdir }}/haproxy_cert/server.crt"
-      flat: yes
-  - fetch:
-      src: /etc/origin/master/master.server.key
-      dest: "{{ sync_tmpdir }}/haproxy_cert/server.key"
-      flat: yes
-
-- name: Compute haproxy_backend_servers and combine certificate
+- name: Compute haproxy_backend_servers
   hosts: localhost
   connection: local
   sudo: false
@@ -234,11 +221,6 @@
   tasks:
   - set_fact:
       haproxy_backend_servers: "{{ hostvars | oo_select_keys(groups['oo_masters_to_config']) | oo_haproxy_backend_masters }}"
-  - shell: cat server.crt server.key > server.pem
-    args:
-      chdir: "{{ g_master_mktemp.stdout }}/haproxy_cert"
-      creates: "{{ g_master_mktemp.stdout }}/haproxy_cert/server.pem"
-
 
 - name: Configure load balancers
   hosts: oo_lb_to_config
@@ -246,32 +228,24 @@
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     haproxy_frontends:
     - name: atomic-openshift-api
+      mode: tcp
       options:
       - tcplog
       binds:
-      - "*:{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }} ssl crt /etc/haproxy/server.pem"
+      - "*:{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}"
       default_backend: atomic-openshift-api
     haproxy_backends:
     - name: atomic-openshift-api
+      mode: tcp
+      option: tcplog
       balance: roundrobin
       servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
-  pre_tasks:
-  - file:
-      path: /etc/haproxy
-      state: directory
-  - copy:
-      src: "{{ sync_tmpdir }}/haproxy_cert/server.pem"
-      dest: /etc/haproxy/server.pem
-      mode: 0600
-      owner: root
-      group: root
   roles:
   - role: haproxy
     when: groups.oo_masters_to_config | length > 1
 
 - name: Configure master instances
   hosts: oo_masters_to_config
-  serial: 1
   vars:
     named_certificates: "{{ hostvars[groups['oo_first_master'][0]]['parsed_named_certificates'] | default([])}}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
diff --git a/roles/haproxy/defaults/main.yml b/roles/haproxy/defaults/main.yml
index 16e9af4d1..7ba5bd485 100644
--- a/roles/haproxy/defaults/main.yml
+++ b/roles/haproxy/defaults/main.yml
@@ -12,3 +12,10 @@ haproxy_backends:
   - name: web01
     address: 127.0.0.1:9000
     opts: check
+
+os_firewall_use_firewalld: False
+os_firewall_allow:
+- service: haproxy stats
+  port: "9000/tcp"
+- service: haproxy balance
+  port: "8443/tcp"
diff --git a/roles/haproxy/meta/main.yml b/roles/haproxy/meta/main.yml
index e02d8f53c..0fad106a9 100644
--- a/roles/haproxy/meta/main.yml
+++ b/roles/haproxy/meta/main.yml
@@ -9,4 +9,6 @@ galaxy_info:
   - name: EL
     versions:
     - 7
-dependencies: []
+dependencies:
+- { role: os_firewall }
+- { role: openshift_repos }
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
index fddf0ede1..c932af72f 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -27,12 +27,17 @@ defaults
     timeout http-request    10s
     timeout queue           1m
     timeout connect         10s
-    timeout client          1m
-    timeout server          1m
+    timeout client          300s
+    timeout server          300s
     timeout http-keep-alive 10s
     timeout check           10s
     maxconn                 3000
 
+listen stats :9000
+    mode http
+    stats enable
+    stats uri /
+
 {% for frontend in haproxy_frontends %}
 frontend  {{ frontend.name }}
 {% for bind in frontend.binds %}
diff --git a/roles/openshift_master/files/atomic-openshift-master-api b/roles/openshift_master/files/atomic-openshift-master-api
new file mode 100644
index 000000000..ea82468a0
--- /dev/null
+++ b/roles/openshift_master/files/atomic-openshift-master-api
@@ -0,0 +1,9 @@
+OPTIONS=
+CONFIG_FILE=/etc/origin/master/master-config.yaml
+
+# Proxy configuration
+# Origin uses standard HTTP_PROXY environment variables. Be sure to set
+# NO_PROXY for your master
+#NO_PROXY=master.example.com
+#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
+#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/files/atomic-openshift-master-api.service b/roles/openshift_master/files/atomic-openshift-master-api.service
new file mode 100644
index 000000000..b24b9809e
--- /dev/null
+++ b/roles/openshift_master/files/atomic-openshift-master-api.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=Atomic OpenShift Master API
+Documentation=https://github.com/openshift/origin
+After=network.target
+After=etcd.service
+Before=atomic-openshift-node.service
+Requires=network.target
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/atomic-openshift-master-api
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/atomic-enterprise start master api --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=131072
+LimitCORE=infinity
+WorkingDirectory=/var/lib/origin/
+SyslogIdentifier=atomic-openshift-master-api
+
+[Install]
+WantedBy=multi-user.target
+WantedBy=atomic-openshift-node.service
diff --git a/roles/openshift_master/files/atomic-openshift-master-controllers b/roles/openshift_master/files/atomic-openshift-master-controllers
new file mode 100644
index 000000000..ea82468a0
--- /dev/null
+++ b/roles/openshift_master/files/atomic-openshift-master-controllers
@@ -0,0 +1,9 @@
+OPTIONS=
+CONFIG_FILE=/etc/origin/master/master-config.yaml
+
+# Proxy configuration
+# Origin uses standard HTTP_PROXY environment variables. Be sure to set
+# NO_PROXY for your master
+#NO_PROXY=master.example.com
+#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
+#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/files/atomic-openshift-master-controllers.service b/roles/openshift_master/files/atomic-openshift-master-controllers.service
new file mode 100644
index 000000000..e84160e5a
--- /dev/null
+++ b/roles/openshift_master/files/atomic-openshift-master-controllers.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=Atomic OpenShift Master Controllers
+Documentation=https://github.com/openshift/origin
+After=network.target
+After=atomic-openshift-master-api.service
+Before=atomic-openshift-node.service
+Requires=network.target
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/atomic-openshift-master-controllers
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/atomic-enterprise start master controllers --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=131072
+LimitCORE=infinity
+WorkingDirectory=/var/lib/origin/
+SyslogIdentifier=atomic-openshift-master-controllers
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+WantedBy=atomic-openshift-node.service
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index b23c19d37..00aaa2e57 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -204,27 +204,16 @@
   when: not openshift_master_ha | bool
   register: start_result
 
-# workaround for start bug when configuring ha
-- name: Start master for ha workaround
-  service: name={{ openshift.common.service_type }}-master state=started
-  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
-
-- name: pause for 30 seconds to let master finish starting up for ha workaround
-  pause: seconds=30
-  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
-
-- name: Stop master for ha workaround
-  service: name={{ openshift.common.service_type }}-master state=stopped
-  when: openshift_master_ha | bool and inventory_hostname in groups.oo_first_master
-# end workaround for start bug when configuring ha
-
-- fail:
-
 - name: Start and enable master api
   service: name={{ openshift.common.service_type }}-master-api enabled=yes state=started
   when: openshift_master_ha | bool
   register: start_result
 
+# TODO: work to eliminate this workaround
+- name: pause a random interval to avoid startup errors for controller
+  pause: seconds={{ 60 | random(step=5) }}
+  when: openshift_master_ha | bool
+
 - name: Start and enable master controller
   service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started
   when: openshift_master_ha | bool
diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml
index abb0f8252..0738048d3 100644
--- a/roles/openshift_master_ca/tasks/main.yml
+++ b/roles/openshift_master_ca/tasks/main.yml
@@ -14,7 +14,7 @@
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ openshift.common.all_hostnames | join(',') }}
+      --hostnames={{ master_hostnames | join(',') }}
       --master={{ openshift.master.api_url }}
       --public-master={{ openshift.master.public_api_url }}
       --cert-dir={{ openshift_master_config_dir }} --overwrite=false
-- 
cgit v1.2.3


From 037d77599632faa81f0c6ca1e912a19f04628b55 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Tue, 25 Aug 2015 15:48:42 -0400
Subject: more tweaks

---
 playbooks/common/openshift-master/config.yml | 1 +
 roles/openshift_master/tasks/main.yml        | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 67068e001..9de72fcde 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -246,6 +246,7 @@
 
 - name: Configure master instances
   hosts: oo_masters_to_config
+  serial: 1
   vars:
     named_certificates: "{{ hostvars[groups['oo_first_master'][0]]['parsed_named_certificates'] | default([])}}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 00aaa2e57..085855750 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -209,15 +209,17 @@
   when: openshift_master_ha | bool
   register: start_result
 
-# TODO: work to eliminate this workaround
-- name: pause a random interval to avoid startup errors for controller
-  pause: seconds={{ 60 | random(step=5) }}
+- name: pause to prevent service restart from interfering with bootstrapping
+  pause: seconds=30
   when: openshift_master_ha | bool
 
+# TODO: fix the ugly workaround of setting ignore_errors
+#       the controllers service tries to start even if it is already started
 - name: Start and enable master controller
   service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started
   when: openshift_master_ha | bool
   register: start_result
+  ignore_errors: yes
 
 - set_fact:
     master_service_status_changed = start_result | changed
-- 
cgit v1.2.3


From 16e085b44ba3e3328ec7d8a5dd5e2331c7f633a6 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Wed, 26 Aug 2015 12:54:29 -0400
Subject: default to source persistence for haproxy

---
 playbooks/common/openshift-master/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 9de72fcde..d1aa9c85c 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -238,7 +238,7 @@
     - name: atomic-openshift-api
       mode: tcp
       option: tcplog
-      balance: roundrobin
+      balance: source
       servers: "{{ hostvars.localhost.haproxy_backend_servers }}"
   roles:
   - role: haproxy
-- 
cgit v1.2.3


From ca9f4f08fbf14f9edfa7331e327cf92a25cd4401 Mon Sep 17 00:00:00 2001
From: Andrew Butcher <abutcher@redhat.com>
Date: Tue, 22 Sep 2015 16:42:36 -0400
Subject: Various HA changes for pacemaker and native methods.

---
 filter_plugins/oo_filters.py                       |  2 +-
 inventory/byo/hosts.example                        | 53 ++++++++++-----
 playbooks/common/openshift-cluster/config.yml      | 10 +--
 .../common/openshift-cluster/evaluate_groups.yml   |  8 +++
 playbooks/common/openshift-master/config.yml       | 54 ++++++++++++++--
 playbooks/gce/openshift-cluster/join_node.yml      |  2 +-
 roles/openshift_common/vars/main.yml               |  2 -
 roles/openshift_facts/library/openshift_facts.py   | 39 ++++++-----
 .../files/atomic-openshift-master-api              |  9 ---
 .../files/atomic-openshift-master-api.service      | 21 ------
 .../files/atomic-openshift-master-controllers      |  9 ---
 .../atomic-openshift-master-controllers.service    | 22 -------
 roles/openshift_master/handlers/main.yml           |  4 +-
 roles/openshift_master/tasks/main.yml              | 75 ++++++++++++++--------
 .../templates/atomic-openshift-master-api.j2       |  9 +++
 .../atomic-openshift-master-api.service.j2         | 21 ++++++
 .../atomic-openshift-master-controllers.j2         |  9 +++
 .../atomic-openshift-master-controllers.service.j2 | 22 +++++++
 .../templates/sessionSecretsFile.yaml.v1.j2        |  7 ++
 roles/openshift_master/vars/main.yml               |  1 +
 .../tasks/configure_deferred.yml                   |  8 ---
 roles/openshift_master_cluster/tasks/main.yml      |  5 +-
 roles/openshift_node/meta/main.yml                 |  1 +
 roles/openshift_node/tasks/main.yml                |  1 +
 roles/openshift_repos/tasks/main.yaml              |  2 +-
 25 files changed, 245 insertions(+), 151 deletions(-)
 delete mode 100644 roles/openshift_master/files/atomic-openshift-master-api
 delete mode 100644 roles/openshift_master/files/atomic-openshift-master-api.service
 delete mode 100644 roles/openshift_master/files/atomic-openshift-master-controllers
 delete mode 100644 roles/openshift_master/files/atomic-openshift-master-controllers.service
 create mode 100644 roles/openshift_master/templates/atomic-openshift-master-api.j2
 create mode 100644 roles/openshift_master/templates/atomic-openshift-master-api.service.j2
 create mode 100644 roles/openshift_master/templates/atomic-openshift-master-controllers.j2
 create mode 100644 roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2
 create mode 100644 roles/openshift_master/templates/sessionSecretsFile.yaml.v1.j2
 delete mode 100644 roles/openshift_master_cluster/tasks/configure_deferred.yml

(limited to 'playbooks/common')

diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index 4e4f7507c..f494c0ae5 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -422,6 +422,6 @@ class FilterModule(object):
             "oo_split": self.oo_split,
             "oo_filter_list": self.oo_filter_list,
             "oo_parse_heat_stack_outputs": self.oo_parse_heat_stack_outputs,
-            "oo_parse_certificate_names": self.oo_parse_certificate_names
+            "oo_parse_certificate_names": self.oo_parse_certificate_names,
             "oo_haproxy_backend_masters": self.oo_haproxy_backend_masters
         }
diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example
index d6c9c3ab6..11f076a8a 100644
--- a/inventory/byo/hosts.example
+++ b/inventory/byo/hosts.example
@@ -21,6 +21,9 @@ ansible_ssh_user=root
 # deployment type valid values are origin, online and enterprise
 deployment_type=atomic-enterprise
 
+# Enable cluster metrics
+#use_cluster_metrics=true
+
 # Pre-release registry URL
 #oreg_url=example.com/openshift3/ose-${component}:${version}
 
@@ -55,27 +58,27 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Set cockpit plugins
 #osm_cockpit_plugins=['cockpit-kubernetes']
 
-# master cluster ha variables using pacemaker or RHEL HA
-#openshift_master_cluster_password=openshift_cluster
-#openshift_master_cluster_vip=192.168.133.25
-#openshift_master_cluster_public_vip=192.168.133.25
+# Native high availbility cluster method with optional load balancer.
+# If no lb group is defined installer assumes that a load balancer has
+# been preconfigured. For installation the value of
+# openshift_master_cluster_hostname must resolve to the load balancer
+# or to one or all of the masters defined in the inventory if no load
+# balancer is present.
+#openshift_master_cluster_method=native
 #openshift_master_cluster_hostname=openshift-ansible.test.example.com
 #openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
 
-# master cluster ha variables when using a different HA solution
-# For installation the value of openshift_master_cluster_hostname must resolve
-# to the first master defined in the inventory.
-# The HA solution must be manually configured after installation and must ensure
-# that the master is running on a single master host.
+# Pacemaker high availability cluster method.
+# Pacemaker HA environment must be able to self provision the
+# configured VIP. For installation openshift_master_cluster_hostname
+# must resolve to the configured VIP.
+#openshift_master_cluster_method=pacemaker
+#openshift_master_cluster_password=openshift_cluster
+#openshift_master_cluster_vip=192.168.133.25
+#openshift_master_cluster_public_vip=192.168.133.25
 #openshift_master_cluster_hostname=openshift-ansible.test.example.com
 #openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
-#openshift_master_cluster_defer_ha=True
 
-# Native clustering with haproxy as an optional load balancer
-#openshift_master_cluster_hostname=openshift-ansible.test.example.com
-#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
-#openshift_master_cluster_vip=192.168.133.25
-#openshift_master_cluster_public_vip=192.168.133.25
 # Override the default controller lease ttl
 #osm_controller_lease_ttl=30
 
@@ -110,6 +113,22 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # Detected names may be overridden by specifying the "names" key
 #openshift_master_named_certificates=[{"certfile": "/path/to/custom1.crt", "keyfile": "/path/to/custom1.key", "names": ["public-master-host.com"]}]
 
+# Session options
+#openshift_master_session_name=ssn
+#openshift_master_session_max_seconds=3600
+
+# An authentication and encryption secret will be generated if secrets
+# are not provided. If provided, openshift_master_session_auth_secrets
+# and openshift_master_encryption_secrets must be equal length.
+#
+# Signing secrets, used to authenticate sessions using
+# HMAC. Recommended to use secrets with 32 or 64 bytes.
+#openshift_master_session_auth_secrets=['DONT+USE+THIS+SECRET+b4NV+pmZNSO']
+#
+# Encrypting secrets, used to encrypt sessions. Must be 16, 24, or 32
+# characters long, to select AES-128, AES-192, or AES-256.
+#openshift_master_session_encryption_secrets=['DONT+USE+THIS+SECRET+b4NV+pmZNSO']
+
 # host group for masters
 [masters]
 ose3-master[1:3]-ansible.test.example.com
@@ -120,7 +139,9 @@ ose3-etcd[1:3]-ansible.test.example.com
 [lb]
 ose3-lb-ansible.test.example.com
 
-# host group for nodes
+# NOTE: Currently we require that masters be part of the SDN which requires that they also be nodes
+# However, in order to ensure that your masters are not burdened with running pods you should
+# make them unschedulable by adding openshift_scheduleable=False any node that's also a master.
 [nodes]
 ose3-master[1:3]-ansible.test.example.com
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index b66ca4709..a8bd634d3 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -1,14 +1,6 @@
 ---
 - include: evaluate_groups.yml
 
-  - name: Evaluate oo_lb_to_config
-    add_host:
-      name: "{{ item }}"
-      groups: oo_lb_to_config
-      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
-      ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: groups[g_lb_group] | default(groups[g_masters_group]) | default([])
-
 - include: ../openshift-etcd/config.yml
 
 - include: ../openshift-master/config.yml
@@ -16,4 +8,4 @@
 - include: ../openshift-node/config.yml
   vars:
     osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
-    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
+    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].cluster_dns_ip }}"
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index 1919660dd..a787ba0d3 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -62,3 +62,11 @@
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
     when: g_masters_group in groups and (groups[g_masters_group] | length) > 0
+
+  - name: Evaluate oo_lb_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_lb_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_lb_group] | default([])
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index d1aa9c85c..64376040f 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -46,7 +46,6 @@
           public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
           cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
           cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
-          cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
           console_path: "{{ openshift_master_console_path | default(None) }}"
           console_port: "{{ openshift_master_console_port | default(None) }}"
           console_url: "{{ openshift_master_console_url | default(None) }}"
@@ -244,6 +243,35 @@
   - role: haproxy
     when: groups.oo_masters_to_config | length > 1
 
+- name: Generate master session keys
+  hosts: oo_first_master
+  tasks:
+  - fail:
+      msg: "Both openshift_master_session_auth_secrets and openshift_master_session_encryption_secrets must be provided if either variable is set"
+    when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is not defined) or (openshift_master_session_encryption_secrets is defined and openshift_master_session_auth_secrets is not defined)
+  - fail:
+      msg: "openshift_master_session_auth_secrets and openshift_master_encryption_secrets must be equal length"
+    when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is defined) and (openshift_master_session_auth_secrets | length != openshift_master_session_encryption_secrets | length)
+  - name: Generate session authentication key
+    command: /usr/bin/openssl rand -base64 24
+    register: session_auth_output
+    with_sequence: count=1
+    when: openshift_master_session_auth_secrets is undefined
+  - name: Generate session encryption key
+    command: /usr/bin/openssl rand -base64 24
+    register: session_encryption_output
+    with_sequence: count=1
+    when: openshift_master_session_encryption_secrets is undefined
+  - set_fact:
+      session_auth_secret: "{{ openshift_master_session_auth_secrets
+                                | default(session_auth_output.results
+                                | map(attribute='stdout')
+                                | list) }}"
+      session_encryption_secret: "{{ openshift_master_session_encryption_secrets
+                                      | default(session_encryption_output.results
+                                      | map(attribute='stdout')
+                                      | list) }}"
+
 - name: Configure master instances
   hosts: oo_masters_to_config
   serial: 1
@@ -252,6 +280,8 @@
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
     openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
+    openshift_master_session_auth_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_auth_secret'] }}"
+    openshift_master_session_encryption_secrets: "{{ hostvars[groups['oo_first_master'][0]]['session_encryption_secret'] }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -276,15 +306,29 @@
 - name: Additional master configuration
   hosts: oo_first_master
   vars:
-  #openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
-  #  omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
   roles:
-#  - role: openshift_master_cluster
-#    when: openshift_master_ha | bool
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool and openshift.master.cluster_method == "pacemaker"
   - openshift_examples
   - role: openshift_cluster_metrics
     when: openshift.common.use_cluster_metrics | bool
 
+- name: Determine cluster dns ip
+  hosts: oo_first_master
+  tasks:
+  - name: Get master service ip
+    command: "{{ openshift.common.client_binary }} get -o template svc kubernetes --template=\\{\\{.spec.clusterIP\\}\\}"
+    register: master_service_ip_output
+    when: openshift.common.version_greater_than_3_1_or_1_1 | bool
+  - set_fact:
+      cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
+    when: not openshift.common.version_greater_than_3_1_or_1_1 | bool
+  - set_fact:
+      cluster_dns_ip: "{{ master_service_ip_output.stdout }}"
+    when: openshift.common.version_greater_than_3_1_or_1_1 | bool
+
 - name: Enable cockpit
   hosts: oo_first_master
   vars:
diff --git a/playbooks/gce/openshift-cluster/join_node.yml b/playbooks/gce/openshift-cluster/join_node.yml
index 0dfa3e9d7..c8f6065cd 100644
--- a/playbooks/gce/openshift-cluster/join_node.yml
+++ b/playbooks/gce/openshift-cluster/join_node.yml
@@ -46,4 +46,4 @@
     openshift_node_labels: "{{ lookup('oo_option', 'openshift_node_labels') }} "
     os_sdn_network_plugin_name: "redhat/openshift-ovs-subnet"
     osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
-    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
+    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].cluster_dns_ip }}"
diff --git a/roles/openshift_common/vars/main.yml b/roles/openshift_common/vars/main.yml
index 817fe0a5f..50816d319 100644
--- a/roles/openshift_common/vars/main.yml
+++ b/roles/openshift_common/vars/main.yml
@@ -5,5 +5,3 @@
 # chains with the public zone (or the zone associated with the correct
 # interfaces)
 os_firewall_use_firewalld: False
-
-openshift_data_dir: /var/lib/origin
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 41c7ff1de..ae5d99121 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -540,7 +540,7 @@ def set_deployment_facts_if_unset(facts):
         if 'service_type' not in facts['common']:
             service_type = 'atomic-openshift'
             if deployment_type == 'origin':
-                service_type = 'openshift'
+                service_type = 'origin'
             elif deployment_type in ['enterprise', 'online']:
                 service_type = 'openshift'
             facts['common']['service_type'] = service_type
@@ -548,23 +548,10 @@ def set_deployment_facts_if_unset(facts):
             config_base = '/etc/origin'
             if deployment_type in ['enterprise', 'online']:
                 config_base = '/etc/openshift'
-            elif deployment_type == 'origin':
-                config_base = '/etc/openshift'
             facts['common']['config_base'] = config_base
         if 'data_dir' not in facts['common']:
             data_dir = '/var/lib/origin'
-            if deployment_type in ['enterprise', 'online']:
-                data_dir = '/var/lib/openshift'
             facts['common']['data_dir'] = data_dir
-        facts['common']['version'] = version = get_openshift_version()
-        if version is not None:
-            if deployment_type == 'origin':
-                version_gt_3_1_or_1_1 = LooseVersion(version) > LooseVersion('1.0.6')
-            else:
-                version_gt_3_1_or_1_1 = LooseVersion(version) > LooseVersion('3.0.2.900')
-        else:
-            version_gt_3_1_or_1_1 = True
-        facts['common']['version_greater_than_3_1_or_1_1'] = version_gt_3_1_or_1_1
 
     for role in ('master', 'node'):
         if role in facts:
@@ -598,6 +585,27 @@ def set_deployment_facts_if_unset(facts):
 
     return facts
 
+def set_version_facts_if_unset(facts):
+    """ Set version facts. This currently includes common.version and
+        common.version_greater_than_3_1_or_1_1.
+
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with version facts.
+    """
+    if 'common' in facts:
+        deployment_type = facts['common']['deployment_type']
+        facts['common']['version'] = version = get_openshift_version()
+        if version is not None:
+            if deployment_type == 'origin':
+                version_gt_3_1_or_1_1 = LooseVersion(version) > LooseVersion('1.0.6')
+            else:
+                version_gt_3_1_or_1_1 = LooseVersion(version) > LooseVersion('3.0.2.900')
+        else:
+            version_gt_3_1_or_1_1 = True
+        facts['common']['version_greater_than_3_1_or_1_1'] = version_gt_3_1_or_1_1
+    return facts
 
 def set_sdn_facts_if_unset(facts):
     """ Set sdn facts if not already present in facts dict
@@ -897,6 +905,7 @@ class OpenShiftFacts(object):
         facts = set_identity_providers_if_unset(facts)
         facts = set_sdn_facts_if_unset(facts)
         facts = set_deployment_facts_if_unset(facts)
+        facts = set_version_facts_if_unset(facts)
         facts = set_aggregate_facts(facts)
         return dict(openshift=facts)
 
@@ -936,7 +945,7 @@ class OpenShiftFacts(object):
                           session_name='ssn', session_secrets_file='',
                           access_token_max_seconds=86400,
                           auth_token_max_seconds=500,
-                          oauth_grant_method='auto', cluster_defer_ha=False)
+                          oauth_grant_method='auto')
             defaults['master'] = master
 
         if 'node' in roles:
diff --git a/roles/openshift_master/files/atomic-openshift-master-api b/roles/openshift_master/files/atomic-openshift-master-api
deleted file mode 100644
index ea82468a0..000000000
--- a/roles/openshift_master/files/atomic-openshift-master-api
+++ /dev/null
@@ -1,9 +0,0 @@
-OPTIONS=
-CONFIG_FILE=/etc/origin/master/master-config.yaml
-
-# Proxy configuration
-# Origin uses standard HTTP_PROXY environment variables. Be sure to set
-# NO_PROXY for your master
-#NO_PROXY=master.example.com
-#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
-#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/files/atomic-openshift-master-api.service b/roles/openshift_master/files/atomic-openshift-master-api.service
deleted file mode 100644
index 4663b77f2..000000000
--- a/roles/openshift_master/files/atomic-openshift-master-api.service
+++ /dev/null
@@ -1,21 +0,0 @@
-[Unit]
-Description=Atomic OpenShift Master API
-Documentation=https://github.com/openshift/origin
-After=network.target
-After=etcd.service
-Before=atomic-openshift-node.service
-Requires=network.target
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/atomic-openshift-master-api
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start master api --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=131072
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=atomic-openshift-master-api
-
-[Install]
-WantedBy=multi-user.target
-WantedBy=atomic-openshift-node.service
diff --git a/roles/openshift_master/files/atomic-openshift-master-controllers b/roles/openshift_master/files/atomic-openshift-master-controllers
deleted file mode 100644
index ea82468a0..000000000
--- a/roles/openshift_master/files/atomic-openshift-master-controllers
+++ /dev/null
@@ -1,9 +0,0 @@
-OPTIONS=
-CONFIG_FILE=/etc/origin/master/master-config.yaml
-
-# Proxy configuration
-# Origin uses standard HTTP_PROXY environment variables. Be sure to set
-# NO_PROXY for your master
-#NO_PROXY=master.example.com
-#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
-#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/files/atomic-openshift-master-controllers.service b/roles/openshift_master/files/atomic-openshift-master-controllers.service
deleted file mode 100644
index 517f9c908..000000000
--- a/roles/openshift_master/files/atomic-openshift-master-controllers.service
+++ /dev/null
@@ -1,22 +0,0 @@
-[Unit]
-Description=Atomic OpenShift Master Controllers
-Documentation=https://github.com/openshift/origin
-After=network.target
-After=atomic-openshift-master-api.service
-Before=atomic-openshift-node.service
-Requires=network.target
-
-[Service]
-Type=notify
-EnvironmentFile=/etc/sysconfig/atomic-openshift-master-controllers
-Environment=GOTRACEBACK=crash
-ExecStart=/usr/bin/openshift start master controllers --config=${CONFIG_FILE} $OPTIONS
-LimitNOFILE=131072
-LimitCORE=infinity
-WorkingDirectory=/var/lib/origin/
-SyslogIdentifier=atomic-openshift-master-controllers
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target
-WantedBy=atomic-openshift-node.service
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index ad3ac5a9f..4b9500cbd 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -5,10 +5,10 @@
 
 - name: restart master api
   service: name={{ openshift.common.service_type }}-master-api state=restarted
-  when: openshift_master_ha | bool
+  when: (openshift_master_ha | bool) and (not master_api_service_status_changed | default(false)) and openshift.master.cluster_method == 'native'
 
 # TODO: need to fix up ignore_errors here
 - name: restart master controllers
   service: name={{ openshift.common.service_type }}-master-controllers state=restarted
-  when: openshift_master_ha | bool
+  when: (openshift_master_ha | bool) and (not master_controllers_service_status_changed | default(false)) and openshift.master.cluster_method == 'native'
   ignore_errors: yes
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 290f22358..be77fce4a 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -8,17 +8,23 @@
     - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods
   when: openshift_master_oauth_grant_method is defined
 
-#- fail:
-#    msg: "openshift_master_cluster_password must be set for multi-master installations"
-#  when: openshift_master_ha | bool and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
+- fail:
+    msg: "openshift_master_cluster_method must be set to either 'native' or 'pacemaker' for multi-master installations"
+  when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method not in ["native", "pacemaker"]))
+- fail:
+    msg: "'native' high availability is not supported for the requested OpenShift version"
+  when: openshift_master_ha | bool and openshift_master_cluster_method == "native" and not openshift.common.version_greater_than_3_1_or_1_1 | bool
+- fail:
+    msg: "openshift_master_cluster_password must be set for multi-master installations"
+  when: openshift_master_ha | bool and openshift_master_cluster_method == "pacemaker" and (openshift_master_cluster_password is not defined or not openshift_master_cluster_password)
 
 - name: Set master facts
   openshift_facts:
     role: master
     local_facts:
+      cluster_method: "{{ openshift_master_cluster_method | default(None) }}"
       cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
       cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
-      cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
       debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}"
       api_port: "{{ openshift_master_api_port | default(None) }}"
       api_url: "{{ openshift_master_api_url | default(None) }}"
@@ -41,6 +47,8 @@
       portal_net: "{{ openshift_master_portal_net | default(None) }}"
       session_max_seconds: "{{ openshift_master_session_max_seconds | default(None) }}"
       session_name: "{{ openshift_master_session_name | default(None) }}"
+      session_auth_secrets: "{{ openshift_master_session_auth_secrets | default(None) }}"
+      session_encryption_secrets: "{{ openshift_master_session_encryption_secrets | default(None) }}"
       session_secrets_file: "{{ openshift_master_session_secrets_file | default(None) }}"
       access_token_max_seconds: "{{ openshift_master_access_token_max_seconds | default(None) }}"
       auth_token_max_seconds: "{{ openshift_master_auth_token_max_seconds | default(None) }}"
@@ -67,7 +75,7 @@
       controller_lease_ttl: "{{ osm_controller_lease_ttl | default(None) }}"
 
 - name: Install Master package
-  yum: pkg={{ openshift.common.service_type }}-master state=present
+  yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version  }} state=present
   register: install_result
 
 # TODO: These values need to be configurable
@@ -79,7 +87,7 @@
       domain: cluster.local
   when: openshift.master.embedded_dns
 
-- name: Create config parent directory if it doesn't exist
+- name: Create config parent directory if it does not exist
   file:
     path: "{{ openshift_master_config_dir }}"
     state: directory
@@ -128,28 +136,37 @@
 
 # workaround for missing systemd unit files for controllers/api
 - name: Create the api service file
-  copy:
-    src: atomic-openshift-master-api.service
-    dest: /usr/lib/systemd/system/atomic-openshift-master-api.service
+  template:
+    src: atomic-openshift-master-api.service.j2
+    dest: /usr/lib/systemd/system/{{ openshift.common.service_type }}-master-api.service
     force: no
 - name: Create the controllers service file
-  copy:
-    src: atomic-openshift-master-controllers.service
-    dest: /usr/lib/systemd/system/atomic-openshift-master-controllers.service
+  template:
+    src: atomic-openshift-master-controllers.service.j2
+    dest: /usr/lib/systemd/system/{{ openshift.common.service_type }}-master-controllers.service
     force: no
 - name: Create the api env file
-  copy:
-    src: atomic-openshift-master-api
-    dest: /etc/sysconfig/atomic-openshift-master-api
+  template:
+    src: atomic-openshift-master-api.j2
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-api
     force: no
 - name: Create the controllers env file
-  copy:
-    src: atomic-openshift-master-controllers
-    dest: /etc/sysconfig/atomic-openshift-master-controllers
+  template:
+    src: atomic-openshift-master-controllers.j2
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-master-controllers
     force: no
 - command: systemctl daemon-reload
 # end workaround for missing systemd unit files
 
+- name: Create session secrets file
+  template:
+    dest: "{{ openshift.master.session_secrets_file }}"
+    src: sessionSecretsFile.yaml.v1.j2
+    force: no
+  notify:
+  - restart master
+  - restart master api
+
 # TODO: add the validate parameter when there is a validation command to run
 - name: Create master config
   template:
@@ -166,6 +183,7 @@
     dest: /etc/sysconfig/{{ openshift.common.service_type }}-master
     regexp: "{{ item.regex }}"
     line: "{{ item.line }}"
+    create: yes
   with_items:
     - regex: '^OPTIONS='
       line: "OPTIONS=--loglevel={{ openshift.master.debug_level }}"
@@ -205,34 +223,39 @@
   when: not openshift_master_ha | bool
   register: start_result
 
+- set_fact:
+    master_service_status_changed = start_result | changed
+  when: not openshift_master_ha | bool
+
 - name: Start and enable master api
   service: name={{ openshift.common.service_type }}-master-api enabled=yes state=started
-  when: openshift_master_ha | bool
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'native'
   register: start_result
 
-- name: pause to prevent service restart from interfering with bootstrapping
-  pause: seconds=30
-  when: openshift_master_ha | bool
+- set_fact:
+    master_api_service_status_changed = start_result | changed
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'native'
 
 # TODO: fix the ugly workaround of setting ignore_errors
 #       the controllers service tries to start even if it is already started
 - name: Start and enable master controller
   service: name={{ openshift.common.service_type }}-master-controllers enabled=yes state=started
-  when: openshift_master_ha | bool
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'native'
   register: start_result
   ignore_errors: yes
 
 - set_fact:
-    master_service_status_changed = start_result | changed
+    master_controllers_service_status_changed = start_result | changed
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'native'
 
 - name: Install cluster packages
   yum: pkg=pcs state=present
-  when: openshift_master_ha | bool and not openshift.master.cluster_defer_ha | bool
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'pacemaker'
   register: install_result
 
 - name: Start and enable cluster service
   service: name=pcsd enabled=yes state=started
-  when: openshift_master_ha | bool and not openshift.master.cluster_defer_ha | bool
+  when: openshift_master_ha | bool and openshift.master.cluster_method == 'pacemaker'
 
 - name: Set the cluster user password
   shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster
diff --git a/roles/openshift_master/templates/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/atomic-openshift-master-api.j2
new file mode 100644
index 000000000..205934248
--- /dev/null
+++ b/roles/openshift_master/templates/atomic-openshift-master-api.j2
@@ -0,0 +1,9 @@
+OPTIONS=
+CONFIG_FILE={{ openshift_master_config_dir }}/master-config.yaml
+
+# Proxy configuration
+# Origin uses standard HTTP_PROXY environment variables. Be sure to set
+# NO_PROXY for your master
+#NO_PROXY=master.example.com
+#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
+#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/templates/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/atomic-openshift-master-api.service.j2
new file mode 100644
index 000000000..ba19fb348
--- /dev/null
+++ b/roles/openshift_master/templates/atomic-openshift-master-api.service.j2
@@ -0,0 +1,21 @@
+[Unit]
+Description=Atomic OpenShift Master API
+Documentation=https://github.com/openshift/origin
+After=network.target
+After=etcd.service
+Before={{ openshift.common.service_type }}-node.service
+Requires=network.target
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-master-api
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/openshift start master api --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=131072
+LimitCORE=infinity
+WorkingDirectory={{ openshift.common.data_dir }}
+SyslogIdentifier=atomic-openshift-master-api
+
+[Install]
+WantedBy=multi-user.target
+WantedBy={{ openshift.common.service_type }}-node.service
diff --git a/roles/openshift_master/templates/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/atomic-openshift-master-controllers.j2
new file mode 100644
index 000000000..205934248
--- /dev/null
+++ b/roles/openshift_master/templates/atomic-openshift-master-controllers.j2
@@ -0,0 +1,9 @@
+OPTIONS=
+CONFIG_FILE={{ openshift_master_config_dir }}/master-config.yaml
+
+# Proxy configuration
+# Origin uses standard HTTP_PROXY environment variables. Be sure to set
+# NO_PROXY for your master
+#NO_PROXY=master.example.com
+#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
+#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
diff --git a/roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2
new file mode 100644
index 000000000..8952c86ef
--- /dev/null
+++ b/roles/openshift_master/templates/atomic-openshift-master-controllers.service.j2
@@ -0,0 +1,22 @@
+[Unit]
+Description=Atomic OpenShift Master Controllers
+Documentation=https://github.com/openshift/origin
+After=network.target
+After={{ openshift.common.service_type }}-master-api.service
+Before={{ openshift.common.service_type }}-node.service
+Requires=network.target
+
+[Service]
+Type=notify
+EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-master-controllers
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/openshift start master controllers --config=${CONFIG_FILE} $OPTIONS
+LimitNOFILE=131072
+LimitCORE=infinity
+WorkingDirectory={{ openshift.common.data_dir }}
+SyslogIdentifier={{ openshift.common.service_type }}-master-controllers
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+WantedBy={{ openshift.common.service_type }}-node.service
diff --git a/roles/openshift_master/templates/sessionSecretsFile.yaml.v1.j2 b/roles/openshift_master/templates/sessionSecretsFile.yaml.v1.j2
new file mode 100644
index 000000000..d12d9db90
--- /dev/null
+++ b/roles/openshift_master/templates/sessionSecretsFile.yaml.v1.j2
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: SessionSecrets
+secrets:
+{% for secret in openshift_master_session_auth_secrets %}
+- authentication: "{{ openshift_master_session_auth_secrets[loop.index0] }}"
+  encryption: "{{ openshift_master_session_encryption_secrets[loop.index0] }}"
+{% endfor %}
diff --git a/roles/openshift_master/vars/main.yml b/roles/openshift_master/vars/main.yml
index ecdb4f883..534465451 100644
--- a/roles/openshift_master/vars/main.yml
+++ b/roles/openshift_master/vars/main.yml
@@ -2,6 +2,7 @@
 openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
 openshift_master_config_file: "{{ openshift_master_config_dir }}/master-config.yaml"
 openshift_master_scheduler_conf: "{{ openshift_master_config_dir }}/scheduler.json"
+openshift_master_session_secrets_file: "{{ openshift_master_config_dir }}/session-secrets.yaml"
 openshift_master_policy: "{{ openshift_master_config_dir }}/policy.json"
 openshift_version: "{{ openshift_pkg_version | default('') }}"
 
diff --git a/roles/openshift_master_cluster/tasks/configure_deferred.yml b/roles/openshift_master_cluster/tasks/configure_deferred.yml
deleted file mode 100644
index 3b416005b..000000000
--- a/roles/openshift_master_cluster/tasks/configure_deferred.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-- debug: msg="Deferring config"
-
-- name: Start and enable the master
-  service:
-    name: "{{ openshift.common.service_type }}-master"
-    state: started
-    enabled: yes
diff --git a/roles/openshift_master_cluster/tasks/main.yml b/roles/openshift_master_cluster/tasks/main.yml
index 315947183..6303a6e46 100644
--- a/roles/openshift_master_cluster/tasks/main.yml
+++ b/roles/openshift_master_cluster/tasks/main.yml
@@ -4,10 +4,7 @@
   register: pcs_status
   changed_when: false
   failed_when: false
-  when: not openshift.master.cluster_defer_ha | bool
+  when: openshift.master.cluster_method == "pacemaker"
 
 - include: configure.yml
   when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr"
-
-- include: configure_deferred.yml
-  when: openshift.master.cluster_defer_ha | bool
diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml
index c92008a77..9d40ae3b3 100644
--- a/roles/openshift_node/meta/main.yml
+++ b/roles/openshift_node/meta/main.yml
@@ -13,3 +13,4 @@ galaxy_info:
   - cloud
 dependencies:
 - { role: openshift_common }
+- { role: docker }
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index c455a09f1..6f163f541 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -68,6 +68,7 @@
     dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
     regexp: "{{ item.regex }}"
     line: "{{ item.line }}"
+    create: yes
   with_items:
     - regex: '^OPTIONS='
       line: "OPTIONS=--loglevel={{ openshift.node.debug_level }}"
diff --git a/roles/openshift_repos/tasks/main.yaml b/roles/openshift_repos/tasks/main.yaml
index 12e98b7a1..aa696ae12 100644
--- a/roles/openshift_repos/tasks/main.yaml
+++ b/roles/openshift_repos/tasks/main.yaml
@@ -8,7 +8,7 @@
 #       proper repos correctly.
 
 - assert:
-    that: openshift_deployment_type in known_openshift_deployment_types
+    that: openshift.common.deployment_type in known_openshift_deployment_types
 
 - name: Ensure libselinux-python is installed
   yum:
-- 
cgit v1.2.3


From c84677a90bd168b804cae7e511747b7af30ce387 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Thu, 5 Nov 2015 21:21:37 -0500
Subject: Fix lb group related errors

---
 playbooks/aws/openshift-cluster/config.yml             |  1 +
 playbooks/common/openshift-cluster/evaluate_groups.yml | 10 +++++++---
 playbooks/gce/openshift-cluster/config.yml             |  1 +
 playbooks/libvirt/openshift-cluster/config.yml         |  1 +
 playbooks/openstack/openshift-cluster/config.yml       |  1 +
 5 files changed, 11 insertions(+), 3 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/aws/openshift-cluster/config.yml b/playbooks/aws/openshift-cluster/config.yml
index a8e3e27bb..5aa6b0f9b 100644
--- a/playbooks/aws/openshift-cluster/config.yml
+++ b/playbooks/aws/openshift-cluster/config.yml
@@ -11,6 +11,7 @@
 - include: ../../common/openshift-cluster/config.yml
   vars:
     g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-lb' }}"
     g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
     g_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index a787ba0d3..2bb69614f 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -4,17 +4,21 @@
   gather_facts: no
   tasks:
   - fail:
-      msg: This playbook rquires g_etcd_group to be set
+      msg: This playbook requires g_etcd_group to be set
     when: g_etcd_group is not defined
 
   - fail:
-      msg: This playbook rquires g_masters_group to be set
+      msg: This playbook requires g_masters_group to be set
     when: g_masters_group is not defined
 
   - fail:
-      msg: This playbook rquires g_nodes_group to be set
+      msg: This playbook requires g_nodes_group to be set
     when: g_nodes_group is not defined
 
+  - fail:
+      msg: This playbook requires g_lb_group to be set
+    when: g_lb_group is not defined
+
   - name: Evaluate oo_etcd_to_config
     add_host:
       name: "{{ item }}"
diff --git a/playbooks/gce/openshift-cluster/config.yml b/playbooks/gce/openshift-cluster/config.yml
index 6ca4f7395..745161bcb 100644
--- a/playbooks/gce/openshift-cluster/config.yml
+++ b/playbooks/gce/openshift-cluster/config.yml
@@ -16,6 +16,7 @@
 - include: ../../common/openshift-cluster/config.yml
   vars:
     g_etcd_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-lb' }}"
     g_masters_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-master' }}"
     g_nodes_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-node' }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
diff --git a/playbooks/libvirt/openshift-cluster/config.yml b/playbooks/libvirt/openshift-cluster/config.yml
index c208eee81..4d1ae22ff 100644
--- a/playbooks/libvirt/openshift-cluster/config.yml
+++ b/playbooks/libvirt/openshift-cluster/config.yml
@@ -15,6 +15,7 @@
 - include: ../../common/openshift-cluster/config.yml
   vars:
     g_etcd_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-lb' }}"
     g_masters_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-master' }}"
     g_nodes_group: "{{ 'tag_env-host-type-' ~ cluster_id ~ '-openshift-node' }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
diff --git a/playbooks/openstack/openshift-cluster/config.yml b/playbooks/openstack/openshift-cluster/config.yml
index a5ee2d6a5..888804e28 100644
--- a/playbooks/openstack/openshift-cluster/config.yml
+++ b/playbooks/openstack/openshift-cluster/config.yml
@@ -10,6 +10,7 @@
 - include: ../../common/openshift-cluster/config.yml
   vars:
     g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-lb' }}"
     g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
     g_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
     g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
-- 
cgit v1.2.3


From c20c024d9729f83607e600d304272398c394652b Mon Sep 17 00:00:00 2001
From: Brenton Leanhardt <bleanhar@redhat.com>
Date: Fri, 6 Nov 2015 09:31:24 -0500
Subject: Bug 1278245 - Failed to add node to existing env using
 atomic-openshift-installer

---
 playbooks/common/openshift-cluster/scaleup.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-cluster/scaleup.yml b/playbooks/common/openshift-cluster/scaleup.yml
index 201320de8..6d2777732 100644
--- a/playbooks/common/openshift-cluster/scaleup.yml
+++ b/playbooks/common/openshift-cluster/scaleup.yml
@@ -1,7 +1,16 @@
 ---
 - include: evaluate_groups.yml
+  vars:
+    g_etcd_group: "{{ 'etcd' }}"
+    g_masters_group: "{{ 'masters' }}"
+    g_nodes_group: "{{ 'nodes' }}"
+    g_lb_group: "{{ 'lb' }}"
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_debug_level: 2
+    openshift_deployment_type: "{{ deployment_type }}"
 
 - include: ../openshift-node/config.yml
   vars:
     osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
     osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
+    openshift_deployment_type: "{{ deployment_type }}"
-- 
cgit v1.2.3


From 53988dfe57a2c061f111cdb3bfe464796a49dba0 Mon Sep 17 00:00:00 2001
From: Andrew Butcher <abutcher@redhat.com>
Date: Sun, 8 Nov 2015 12:34:23 -0500
Subject: Conditionals for flannel etcd client certs.

---
 playbooks/common/openshift-node/config.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index ba96b4a78..8da9e231f 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -45,6 +45,7 @@
     - node.etcd-client.crt
     - node.etcd-ca.crt
     register: g_external_etcd_flannel_cert_stat_result
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config and (openshift.common.use_flannel | bool)
   - set_fact:
       etcd_client_flannel_certs_missing: "{{ g_external_etcd_flannel_cert_stat_result.results
                                              | map(attribute='stat.exists')
@@ -69,11 +70,13 @@
   hosts: oo_first_etcd
   vars:
     etcd_generated_certs_dir: /etc/etcd/generated_certs
-    etcd_needing_client_certs: "{{ hostvars
-                                   | oo_select_keys(groups['oo_nodes_to_config'])
-                                   | oo_filter_list(filter_attr='etcd_client_flannel_certs_missing') }}"
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
   pre_tasks:
+  - set_fact:
+      etcd_needing_client_certs: "{{ hostvars
+                                   | oo_select_keys(groups['oo_nodes_to_config'])
+                                   | oo_filter_list(filter_attr='etcd_client_flannel_certs_missing') | default([]) }}"
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
   roles:
   - role: etcd_certificates
   post_tasks:
@@ -84,6 +87,7 @@
     args:
       creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
     with_items: etcd_needing_client_certs
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
   - name: Retrieve the etcd cert tarballs
     fetch:
       src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
@@ -92,6 +96,7 @@
       fail_on_missing: yes
       validate_checksum: yes
     with_items: etcd_needing_client_certs
+    when: etcd_client_flannel_certs_missing is defined and etcd_client_flannel_certs_missing
 
 - name: Copy the external etcd flannel certs to the nodes
   hosts: oo_nodes_to_config
-- 
cgit v1.2.3


From 4c1b0dd4ab8f3a5d4fcfa4ba1501ed374793e77a Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Fri, 6 Nov 2015 16:56:37 -0500
Subject: Refactor upgrade playbook(s)

- Split playbooks into two, one for 3.0 minor upgrades and one for 3.0 to 3.1
  upgrades
- Move upgrade playbooks to common/openshift/cluster/upgrades from adhoc
- Added a byo wrapper playbooks to set the groups based on the byo
  conventions, other providers will need similar playbooks added eventually
- installer wrapper updates for refactored upgrade playbooks
  - call new 3.0 to 3.1 upgrade playbook
  - various fixes for edge cases I hit with a really old config laying
    around.
  - fix output of host facts to show connect_to value.
---
 playbooks/adhoc/upgrades/README.md                 |  21 --
 playbooks/adhoc/upgrades/files/pre-upgrade-check   | 188 ----------
 playbooks/adhoc/upgrades/files/versions.sh         |  10 -
 playbooks/adhoc/upgrades/filter_plugins            |   1 -
 .../upgrades/library/openshift_upgrade_config.py   | 154 --------
 playbooks/adhoc/upgrades/lookup_plugins            |   1 -
 playbooks/adhoc/upgrades/roles                     |   1 -
 playbooks/adhoc/upgrades/upgrade.yml               | 407 ---------------------
 playbooks/byo/openshift-cluster/upgrades/README.md |   8 +
 .../upgrades/v3_0_minor/README.md                  |  21 ++
 .../upgrades/v3_0_minor/upgrade.yml                |   9 +
 .../upgrades/v3_0_to_v3_1/README.md                |  17 +
 .../upgrades/v3_0_to_v3_1/upgrade.yml              |   9 +
 .../upgrades/files/pre-upgrade-check               | 188 ++++++++++
 .../openshift-cluster/upgrades/files/versions.sh   |  10 +
 .../openshift-cluster/upgrades/filter_plugins      |   1 +
 .../upgrades/library/openshift_upgrade_config.py   | 154 ++++++++
 .../openshift-cluster/upgrades/lookup_plugins      |   1 +
 playbooks/common/openshift-cluster/upgrades/roles  |   1 +
 .../upgrades/v3_0_minor/filter_plugins             |   1 +
 .../openshift-cluster/upgrades/v3_0_minor/library  |   1 +
 .../upgrades/v3_0_minor/lookup_plugins             |   1 +
 .../openshift-cluster/upgrades/v3_0_minor/roles    |   1 +
 .../upgrades/v3_0_minor/upgrade.yml                | 112 ++++++
 .../upgrades/v3_0_to_v3_1/filter_plugins           |   1 +
 .../upgrades/v3_0_to_v3_1/library                  |   1 +
 .../upgrades/v3_0_to_v3_1/lookup_plugins           |   1 +
 .../openshift-cluster/upgrades/v3_0_to_v3_1/roles  |   1 +
 .../upgrades/v3_0_to_v3_1/upgrade.yml              | 407 +++++++++++++++++++++
 playbooks/common/openshift-etcd/config.yml         |   2 +
 playbooks/common/openshift-master/config.yml       |   3 +
 roles/openshift_facts/library/openshift_facts.py   |  71 ++--
 utils/src/ooinstall/cli_installer.py               |   5 +-
 utils/src/ooinstall/oo_config.py                   |   8 +-
 utils/src/ooinstall/openshift_ansible.py           |   4 +-
 35 files changed, 998 insertions(+), 824 deletions(-)
 delete mode 100644 playbooks/adhoc/upgrades/README.md
 delete mode 100644 playbooks/adhoc/upgrades/files/pre-upgrade-check
 delete mode 100644 playbooks/adhoc/upgrades/files/versions.sh
 delete mode 120000 playbooks/adhoc/upgrades/filter_plugins
 delete mode 100755 playbooks/adhoc/upgrades/library/openshift_upgrade_config.py
 delete mode 120000 playbooks/adhoc/upgrades/lookup_plugins
 delete mode 120000 playbooks/adhoc/upgrades/roles
 delete mode 100644 playbooks/adhoc/upgrades/upgrade.yml
 create mode 100644 playbooks/byo/openshift-cluster/upgrades/README.md
 create mode 100644 playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md
 create mode 100644 playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
 create mode 100644 playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md
 create mode 100644 playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
 create mode 100644 playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
 create mode 100644 playbooks/common/openshift-cluster/upgrades/files/versions.sh
 create mode 120000 playbooks/common/openshift-cluster/upgrades/filter_plugins
 create mode 100755 playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
 create mode 120000 playbooks/common/openshift-cluster/upgrades/lookup_plugins
 create mode 120000 playbooks/common/openshift-cluster/upgrades/roles
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_minor/library
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles
 create mode 100644 playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins
 create mode 120000 playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles
 create mode 100644 playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml

(limited to 'playbooks/common')

diff --git a/playbooks/adhoc/upgrades/README.md b/playbooks/adhoc/upgrades/README.md
deleted file mode 100644
index 6de8a970f..000000000
--- a/playbooks/adhoc/upgrades/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# [NOTE]
-This playbook will re-run installation steps overwriting any local
-modifications. You should ensure that your inventory has been updated with any
-modifications you've made after your initial installation. If you find any items
-that cannot be configured via ansible please open an issue at
-https://github.com/openshift/openshift-ansible
-
-# Overview
-This playbook is available as a technical preview. It currently performs the
-following steps.
-
- * Upgrade and restart master services
- * Upgrade and restart node services
- * Applies latest configuration by re-running the installation playbook
- * Applies the latest cluster policies
- * Updates the default router if one exists
- * Updates the default registry if one exists
- * Updates image streams and quickstarts
-
-# Usage
-ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/adhoc/upgrades/upgrade.yml
diff --git a/playbooks/adhoc/upgrades/files/pre-upgrade-check b/playbooks/adhoc/upgrades/files/pre-upgrade-check
deleted file mode 100644
index ed4ab6d1b..000000000
--- a/playbooks/adhoc/upgrades/files/pre-upgrade-check
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-"""
-Pre-upgrade checks that must be run on a master before proceeding with upgrade.
-"""
-# This is a script not a python module:
-# pylint: disable=invalid-name
-
-# NOTE: This script should not require any python libs other than what is
-# in the standard library.
-
-__license__ = "ASL 2.0"
-
-import json
-import os
-import subprocess
-import re
-
-# The maximum length of container.ports.name
-ALLOWED_LENGTH = 15
-# The valid structure of container.ports.name
-ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$')
-AT_LEAST_ONE_LETTER = re.compile('[a-z]')
-# look at OS_PATH for the full path. Default ot 'oc'
-OC_PATH = os.getenv('OC_PATH', 'oc')
-
-
-def validate(value):
-    """
-    validate verifies that value matches required conventions
-
-    Rules of container.ports.name validation:
-
-    * must be less that 16 chars
-    * at least one letter
-    * only a-z0-9-
-    * hyphens can not be leading or trailing or next to each other
-
-    :Parameters:
-       - `value`: Value to validate
-    """
-    if len(value) > ALLOWED_LENGTH:
-        return False
-
-    if '--' in value:
-        return False
-
-    # We search since it can be anywhere
-    if not AT_LEAST_ONE_LETTER.search(value):
-        return False
-
-    # We match because it must start at the beginning
-    if not ALLOWED_CHARS.match(value):
-        return False
-    return True
-
-
-def list_items(kind):
-    """
-    list_items returns a list of items from the api
-
-    :Parameters:
-       - `kind`: Kind of item to access
-    """
-    response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind])
-    items = json.loads(response)
-    return items.get("items", [])
-
-
-def get(obj, *paths):
-    """
-    Gets an object
-
-    :Parameters:
-       - `obj`: A dictionary structure
-       - `path`: All other non-keyword arguments
-    """
-    ret_obj = obj
-    for path in paths:
-        if ret_obj.get(path, None) is None:
-            return []
-        ret_obj = ret_obj[path]
-    return ret_obj
-
-
-# pylint: disable=too-many-arguments
-def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid):
-    """
-    Prints out results in human friendly way.
-
-    :Parameters:
-       - `namespace`: Namespace of the resource
-       - `kind`: Kind of the resource
-       - `item_name`: Name of the resource
-       - `container_name`: Name of the container. May be "" when kind=Service.
-       - `port_name`: Name of the port
-       - `valid`: True if the port is valid
-    """
-    if not valid:
-        if len(container_name) > 0:
-            print('%s/%s -n %s (Container="%s" Port="%s")' % (
-                kind, item_name, namespace, container_name, port_name))
-        else:
-            print('%s/%s -n %s (Port="%s")' % (
-                kind, item_name, namespace, port_name))
-
-
-def print_validation_header():
-    """
-    Prints the error header. Should run on the first error to avoid
-    overwhelming the user.
-    """
-    print """\
-At least one port name does not validate. Valid port names:
-
-    * must be less that 16 chars
-    * have at least one letter
-    * only a-z0-9-
-    * do not start or end with -
-    * Dashes may not be next to eachother ('--')
-"""
-
-
-def main():
-    """
-    main is the main entry point to this script
-    """
-    try:
-        # the comma at the end suppresses the newline
-        print "Checking for oc ...",
-        subprocess.check_output([OC_PATH, 'whoami'])
-        print "found"
-    except:
-        print(
-            'Unable to run "%s whoami"\n'
-            'Please ensure OpenShift is running, and "oc" is on your system '
-            'path.\n'
-            'You can override the path with the OC_PATH environment variable.'
-            % OC_PATH)
-        raise SystemExit(1)
-
-    # Where the magic happens
-    first_error = True
-    for kind, path in [
-            ('replicationcontrollers', ("spec", "template", "spec", "containers")),
-            ('pods', ("spec", "containers")),
-            ('deploymentconfigs', ("spec", "template", "spec", "containers"))]:
-        for item in list_items(kind):
-            namespace = item["metadata"]["namespace"]
-            item_name = item["metadata"]["name"]
-            for container in get(item, *path):
-                container_name = container["name"]
-                for port in get(container, "ports"):
-                    port_name = port.get("name", None)
-                    if not port_name:
-                        # Unnamed ports are OK
-                        continue
-                    valid = validate(port_name)
-                    if not valid and first_error:
-                        first_error = False
-                        print_validation_header()
-                    pretty_print_errors(
-                        namespace, kind, item_name,
-                        container_name, port_name, valid)
-
-    # Services follow a different flow
-    for item in list_items('services'):
-        namespace = item["metadata"]["namespace"]
-        item_name = item["metadata"]["name"]
-        for port in get(item, "spec", "ports"):
-            port_name = port.get("targetPort", None)
-            if isinstance(port_name, int) or port_name is None:
-                # Integer only or unnamed ports are OK
-                continue
-            valid = validate(port_name)
-            if not valid and first_error:
-                first_error = False
-                print_validation_header()
-            pretty_print_errors(
-                namespace, "services", item_name, "", port_name, valid)
-
-    # If we had at least 1 error then exit with 1
-    if not first_error:
-        raise SystemExit(1)
-
-
-if __name__ == '__main__':
-    main()
-
diff --git a/playbooks/adhoc/upgrades/files/versions.sh b/playbooks/adhoc/upgrades/files/versions.sh
deleted file mode 100644
index f90719cab..000000000
--- a/playbooks/adhoc/upgrades/files/versions.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
-
-yum_available=$(yum list available "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
-
-
-echo "---"
-echo "curr_version: ${yum_installed}" 
-echo "avail_version: ${yum_available}"
diff --git a/playbooks/adhoc/upgrades/filter_plugins b/playbooks/adhoc/upgrades/filter_plugins
deleted file mode 120000
index b0b7a3414..000000000
--- a/playbooks/adhoc/upgrades/filter_plugins
+++ /dev/null
@@ -1 +0,0 @@
-../../../filter_plugins/
\ No newline at end of file
diff --git a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py b/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py
deleted file mode 100755
index a6721bb92..000000000
--- a/playbooks/adhoc/upgrades/library/openshift_upgrade_config.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-# vim: expandtab:tabstop=4:shiftwidth=4
-
-"""Ansible module for modifying OpenShift configs during an upgrade"""
-
-import os
-import yaml
-
-DOCUMENTATION = '''
----
-module: openshift_upgrade_config
-short_description: OpenShift Upgrade Config
-author: Jason DeTiberus
-requirements: [ ]
-'''
-EXAMPLES = '''
-'''
-
-def modify_api_levels(level_list, remove, ensure, msg_prepend='',
-                      msg_append=''):
-    """ modify_api_levels """
-    changed = False
-    changes = []
-
-    if not isinstance(remove, list):
-        remove = []
-
-    if not isinstance(ensure, list):
-        ensure = []
-
-    if not isinstance(level_list, list):
-        new_list = []
-        changed = True
-        changes.append("%s created missing %s" % (msg_prepend, msg_append))
-    else:
-        new_list = level_list
-        for level in remove:
-            if level in new_list:
-                new_list.remove(level)
-                changed = True
-                changes.append("%s removed %s %s" % (msg_prepend, level, msg_append))
-
-    for level in ensure:
-        if level not in new_list:
-            new_list.append(level)
-            changed = True
-            changes.append("%s added %s %s" % (msg_prepend, level, msg_append))
-
-    return {'new_list': new_list, 'changed': changed, 'changes': changes}
-
-
-def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup):
-    """Main upgrade method for 3.0 to 3.1."""
-    changes = []
-
-    # Facts do not get transferred to the hosts where custom modules run,
-    # need to make some assumptions here.
-    master_config = os.path.join(config_base, 'master/master-config.yaml')
-
-    master_cfg_file = open(master_config, 'r')
-    config = yaml.safe_load(master_cfg_file.read())
-    master_cfg_file.close()
-
-
-    # Remove unsupported api versions and ensure supported api versions from
-    # master config
-    unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3']
-    supported_levels = ['v1']
-
-    result = modify_api_levels(config.get('apiLevels'), unsupported_levels,
-                               supported_levels, 'master-config.yaml:', 'from apiLevels')
-    if result['changed']:
-        config['apiLevels'] = result['new_list']
-        changes.append(result['changes'])
-
-    if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']:
-        config['kubernetesMasterConfig'].pop('apiLevels')
-        changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels')
-
-    # Add proxyClientInfo to master-config
-    if 'proxyClientInfo' not in config['kubernetesMasterConfig']:
-        config['kubernetesMasterConfig']['proxyClientInfo'] = {
-            'certFile': 'master.proxy-client.crt',
-            'keyFile': 'master.proxy-client.key'
-        }
-        changes.append("master-config.yaml: added proxyClientInfo")
-
-    if len(changes) > 0:
-        if backup:
-            # TODO: Check success:
-            ansible_module.backup_local(master_config)
-
-        # Write the modified config:
-        out_file = open(master_config, 'w')
-        out_file.write(yaml.safe_dump(config, default_flow_style=False))
-        out_file.close()
-
-    return changes
-
-
-def upgrade_master(ansible_module, config_base, from_version, to_version, backup):
-    """Upgrade entry point."""
-    if from_version == '3.0':
-        if to_version == '3.1':
-            return upgrade_master_3_0_to_3_1(ansible_module, config_base, backup)
-
-
-def main():
-    """ main """
-    # disabling pylint errors for global-variable-undefined and invalid-name
-    # for 'global module' usage, since it is required to use ansible_facts
-    # pylint: disable=global-variable-undefined, invalid-name,
-    # redefined-outer-name
-    global module
-
-    module = AnsibleModule(
-        argument_spec=dict(
-            config_base=dict(required=True),
-            from_version=dict(required=True, choices=['3.0']),
-            to_version=dict(required=True, choices=['3.1']),
-            role=dict(required=True, choices=['master']),
-            backup=dict(required=False, default=True, type='bool')
-        ),
-        supports_check_mode=True,
-    )
-
-    from_version = module.params['from_version']
-    to_version = module.params['to_version']
-    role = module.params['role']
-    backup = module.params['backup']
-    config_base = module.params['config_base']
-
-    try:
-        changes = []
-        if role == 'master':
-            changes = upgrade_master(module, config_base, from_version,
-                                     to_version, backup)
-
-        changed = len(changes) > 0
-        return module.exit_json(changed=changed, changes=changes)
-
-    # ignore broad-except error to avoid stack trace to ansible user
-    # pylint: disable=broad-except
-    except Exception, e:
-        return module.fail_json(msg=str(e))
-
-# ignore pylint errors related to the module_utils import
-# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
-# import module snippets
-from ansible.module_utils.basic import *
-
-if __name__ == '__main__':
-    main()
diff --git a/playbooks/adhoc/upgrades/lookup_plugins b/playbooks/adhoc/upgrades/lookup_plugins
deleted file mode 120000
index 73cafffe5..000000000
--- a/playbooks/adhoc/upgrades/lookup_plugins
+++ /dev/null
@@ -1 +0,0 @@
-../../../lookup_plugins/
\ No newline at end of file
diff --git a/playbooks/adhoc/upgrades/roles b/playbooks/adhoc/upgrades/roles
deleted file mode 120000
index e2b799b9d..000000000
--- a/playbooks/adhoc/upgrades/roles
+++ /dev/null
@@ -1 +0,0 @@
-../../../roles/
\ No newline at end of file
diff --git a/playbooks/adhoc/upgrades/upgrade.yml b/playbooks/adhoc/upgrades/upgrade.yml
deleted file mode 100644
index 324f5fba3..000000000
--- a/playbooks/adhoc/upgrades/upgrade.yml
+++ /dev/null
@@ -1,407 +0,0 @@
----
-- name: Load master facts
-  hosts: masters
-  roles:
-  - openshift_facts
-
-- name: Verify upgrade can proceed
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  gather_facts: no
-  tasks:
-    # Pacemaker is currently the only supported upgrade path for multiple masters
-    - fail:
-        msg: "openshift_master_cluster_method must be set to 'pacemaker'"
-      when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker"))
-
-- name: Run pre-upgrade checks on first master
-  hosts: masters[0]
-  tasks:
-  # If this script errors out ansible will show the default stdout/stderr
-  # which contains details for the user:
-  - script: files/pre-upgrade-check
-
-- name: Evaluate etcd_hosts
-  hosts: localhost
-  tasks:
-  - name: Evaluate etcd hosts
-    add_host:
-      name: "{{ groups.masters.0 }}"
-      groups: etcd_hosts
-    when: hostvars[groups.masters.0].openshift.master.embedded_etcd | bool
-  - name: Evaluate etcd hosts
-    add_host:
-      name: "{{ item }}"
-      groups: etcd_hosts
-    with_items: groups.etcd
-    when: not hostvars[groups.masters.0].openshift.master.embedded_etcd | bool
-
-- name: Backup etcd
-  hosts: etcd_hosts
-  vars:
-    embedded_etcd: "{{ openshift.master.embedded_etcd }}"
-    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
-  roles:
-  - openshift_facts
-  tasks:
-
-  - stat: path=/var/lib/openshift
-    register: var_lib_openshift
-
-  - stat: path=/var/lib/origin
-    register: var_lib_origin
-
-  - name: Create origin symlink if necessary
-    file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
-    when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
-
-  - name: Check available disk space for etcd backup
-    # We assume to be using the data dir for all backups.
-    shell: >
-      df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
-    register: avail_disk
-
-  - name: Check current embedded etcd disk usage
-    shell: >
-      du -k {{ openshift.master.etcd_data_dir }} | tail -n 1 | cut -f1
-    register: etcd_disk_usage
-    when: embedded_etcd | bool
-
-  - name: Abort if insufficient disk space for etcd backup
-    fail:
-      msg: >
-        {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
-        {{ avail_disk.stdout }} Kb available.
-    when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
-
-  - name: Install etcd (for etcdctl)
-    yum:
-      pkg: etcd
-      state: latest
-
-  - name: Generate etcd backup
-    command: >
-      etcdctl backup --data-dir={{ openshift.master.etcd_data_dir }}
-      --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
-
-  - name: Display location of etcd backup
-    debug:
-      msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
-
-- name: Update deployment type
-  hosts: OSEv3
-  roles:
-  - openshift_facts
-  post_tasks:
-  - openshift_facts:
-      role: common
-      local_facts:
-        deployment_type: "{{ deployment_type }}"
-
-
-- name: Perform upgrade version checking
-  hosts: masters[0]
-  tasks:
-  - name: Clean yum cache
-    command: yum clean all
-
-  - name: Determine available versions
-    script: files/versions.sh {{ openshift.common.service_type }} openshift
-    register: g_versions_result
-
-  - set_fact:
-      g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}"
-
-  - set_fact:
-      g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
-
-  - fail:
-      msg: This playbook requires Origin 1.0.6 or later
-    when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
-
-  # TODO: This should be specific to the 3.1 upgrade playbook (coming in future refactor), otherwise we are blocking 3.0.1 to 3.0.2 here.
-  - fail:
-      msg: Atomic OpenShift 3.1 packages not found
-    when: deployment_type in ['openshift-enterprise', 'atomic-openshift'] and g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
-  # Deployment type 'enterprise' is no longer valid if we're upgrading to 3.1 or beyond.
-  # (still valid for 3.0.x to 3.0.y however) Using the global deployment_type here as
-  # we're checking what was requested by the upgrade, not the current type on the system.
-  - fail:
-      msg: "Deployment type enterprise not supported for upgrade"
-    when: deployment_type == "enterprise" and  g_aos_versions.curr_version | version_compare('3.1', '>=')
-
-
-- name: Upgrade masters
-  hosts: masters
-  vars:
-    openshift_version: "{{ openshift_pkg_version | default('') }}"
-  tasks:
-    - name: Upgrade to latest available kernel
-      yum:
-        pkg: kernel
-        state: latest
-
-    - name: Upgrade master packages
-      command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
-
-    - name: Ensure python-yaml present for config upgrade
-      yum:
-        pkg: PyYAML
-        state: installed
-
-    - name: Upgrade master configuration
-      openshift_upgrade_config:
-        from_version: '3.0'
-        to_version: '3.1'
-        role: master
-        config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}"
-      when: deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_aos_versions.curr_version | version_compare('3.1', '>=')
-
-    - set_fact:
-        master_certs_missing: True
-        master_cert_subdir: master-{{ openshift.common.hostname }}
-        master_cert_config_dir: "{{ openshift.common.config_base }}/master"
-
-- name: Create temp directory for syncing certs
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - name: Create local temp directory for syncing certs
-    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
-    register: g_master_mktemp
-    changed_when: False
-
-- name: Generate missing master certificates
-  hosts: masters[0]
-  vars:
-    master_hostnames: "{{ hostvars
-                          | oo_select_keys(groups.masters)
-                          | oo_collect('openshift.common.all_hostnames')
-                          | oo_flatten | unique }}"
-    master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs"
-    masters_needing_certs: "{{ hostvars
-                               | oo_select_keys(groups.masters)
-                               | difference([groups.masters.0]) }}"
-    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-    openshift_deployment_type: "{{ deployment_type }}"
-  roles:
-  - openshift_master_certificates
-  post_tasks:
-  - name: Remove generated etcd client certs when using external etcd
-    file:
-      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
-      state: absent
-    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
-    with_nested:
-    - masters_needing_certs
-    - - master.etcd-client.crt
-      - master.etcd-client.key
-
-  - name: Create a tarball of the master certs
-    command: >
-      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
-        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
-    with_items: masters_needing_certs
-
-  - name: Retrieve the master cert tarball from the master
-    fetch:
-      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
-      dest: "{{ sync_tmpdir }}/"
-      flat: yes
-      fail_on_missing: yes
-      validate_checksum: yes
-    with_items: masters_needing_certs
-
-- name: Sync certs and restart masters post configuration change
-  hosts: masters
-  vars:
-    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-  - name: Unarchive the tarball on the master
-    unarchive:
-      src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
-      dest: "{{ master_cert_config_dir }}"
-    when: inventory_hostname != groups.masters.0
-
-  - name: Restart master services
-    service: name="{{ openshift.common.service_type}}-master" state=restarted
-    when: not openshift_master_ha | bool
-
-- name: Destroy cluster
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-    openshift_deployment_type: "{{ deployment_type }}"
-  pre_tasks:
-  - name: Check for configured cluster
-    stat:
-      path: /etc/corosync/corosync.conf
-    register: corosync_conf
-    when: openshift_master_ha | bool
-  - name: Destroy cluster
-    command: pcs cluster destroy --all
-    when: openshift_master_ha | bool and corosync_conf.stat.exists == true
-
-- name: Start pcsd on masters
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-  - name: Start pcsd
-    service: name=pcsd enabled=yes state=started
-    when: openshift_master_ha | bool
-
-- name: Re-create cluster
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-    openshift_deployment_type: "{{ deployment_type }}"
-    omc_cluster_hosts: "{{ groups.masters | join(' ') }}"
-  roles:
-  - role: openshift_master_cluster
-    when: openshift_master_ha | bool
-
-- name: Delete temporary directory on localhost
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - file: name={{ g_master_mktemp.stdout }} state=absent
-    changed_when: False
-
-
-- name: Upgrade nodes
-  hosts: nodes
-  vars:
-    openshift_version: "{{ openshift_pkg_version | default('') }}"
-  roles:
-  - openshift_facts
-  tasks:
-    - name: Upgrade node packages
-      command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }}
-    - name: Restart node services
-      service: name="{{ openshift.common.service_type }}-node" state=restarted
-
-- name: Update cluster policy and policy bindings
-  hosts: masters[0]
-  vars:
-    origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}"
-    ent_reconcile_bindings: "{{ deployment_type in ['openshift-enterprise', 'atomic-enterprise'] and g_new_version | version_compare('3.0.2','>') }}"
-  tasks:
-    - name: oadm policy reconcile-cluster-roles --confirm
-      command: >
-        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
-        policy reconcile-cluster-roles --confirm
-
-    - name: oadm policy reconcile-cluster-role-bindings --confirm
-      command: >
-        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
-        policy reconcile-cluster-role-bindings
-        --exclude-groups=system:authenticated
-        --exclude-groups=system:unauthenticated
-        --exclude-users=system:anonymous
-        --additive-only=true --confirm
-      when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
-
-
-- name: Restart masters post reconcile
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-    - name: Restart master services
-      service: name="{{ openshift.common.service_type}}-master" state=restarted
-      when: not openshift_master_ha | bool
-
-- name: Restart cluster post reconcile
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-    - name: Restart master cluster
-      command: pcs resource restart master
-      when: openshift_master_ha | bool
-    - name: Wait for the clustered master service to be available
-      wait_for:
-        host: "{{ openshift_master_cluster_vip }}"
-        port: 8443
-        state: started
-        timeout: 180
-        delay: 90
-      when: openshift_master_ha | bool
-
-- name: Upgrade default router and registry
-  hosts: masters[0]
-  vars:
-    - registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + g_new_version  ) }}"
-    - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
-    - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
-  tasks:
-    - name: Check for default router
-      command: >
-        {{ oc_cmd }} get -n default dc/router
-      register: _default_router
-      failed_when: false
-      changed_when: false
-    - name: Check for allowHostNetwork and allowHostPorts
-      when: _default_router.rc == 0
-      shell: >
-        {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork
-      register: _scc
-    - name: Grant allowHostNetwork and allowHostPorts
-      when:
-        - _default_router.rc == 0
-        - "'false' in _scc.stdout"
-      command: >
-        {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
-    - name: Update deployment config to 1.0.4/3.0.1 spec
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p
-        '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
-    - name: Switch to hostNetwork=true
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
-    - name: Update router image to current version
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p
-        '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
-
-    - name: Check for default registry
-      command: >
-          {{ oc_cmd }} get -n default dc/docker-registry
-      register: _default_registry
-      failed_when: false
-      changed_when: false
-    - name: Update registry image to current version
-      when: _default_registry.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/docker-registry -p
-        '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
-
-- name: Update image streams and templates
-  hosts: masters[0]
-  vars:
-    openshift_examples_import_command: "update"
-    openshift_deployment_type: "{{ deployment_type }}"
-  roles:
-    - openshift_examples
-
-- name: Ensure master services enabled
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-  - name: Enable master services
-    service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes
-    when: not openshift_master_ha | bool
-
-- name: Ensure node services enabled
-  hosts: nodes
-  tasks:
-    - name: Restart node services
-      service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
-
diff --git a/playbooks/byo/openshift-cluster/upgrades/README.md b/playbooks/byo/openshift-cluster/upgrades/README.md
new file mode 100644
index 000000000..ce7aebf8e
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/README.md
@@ -0,0 +1,8 @@
+# Upgrade playbooks
+The playbooks provided in this directory can be used for upgrading an existing
+environment. Additional notes for the associated upgrade playbooks are
+provided in their respective directories.
+
+# Upgrades available
+- [OpenShift Enterprise 3.0 to latest minor release](v3_0_minor/README.md)
+- [OpenShift Enterprise 3.0 to 3.1](v3_0_to_v3_1/README.md)
diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md
new file mode 100644
index 000000000..c91a6cb96
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/README.md
@@ -0,0 +1,21 @@
+# v3.0 minor upgrade playbook
+**Note:** This playbook will re-run installation steps overwriting any local
+modifications. You should ensure that your inventory has been updated with any
+modifications you've made after your initial installation. If you find any items
+that cannot be configured via ansible please open an issue at
+https://github.com/openshift/openshift-ansible
+
+## Overview
+This playbook is available as a technical preview. It currently performs the
+following steps.
+
+ * Upgrade and restart master services
+ * Upgrade and restart node services
+ * Applies latest configuration by re-running the installation playbook
+ * Applies the latest cluster policies
+ * Updates the default router if one exists
+ * Updates the default registry if one exists
+ * Updates image streams and quickstarts
+
+## Usage
+ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
new file mode 100644
index 000000000..76fa9ba22
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
@@ -0,0 +1,9 @@
+---
+- include: ../../../../common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
+  vars:
+    g_etcd_group: "{{ 'etcd' }}"
+    g_masters_group: "{{ 'masters' }}"
+    g_nodes_group: "{{ 'nodes' }}"
+    g_lb_group: "{{ 'lb' }}"
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md
new file mode 100644
index 000000000..c434be5b7
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/README.md
@@ -0,0 +1,17 @@
+# v3.0 to v3.1 upgrade playbook
+
+## Overview
+This playbook currently performs the
+following steps.
+
+**TODO: update for current steps**
+ * Upgrade and restart master services
+ * Upgrade and restart node services
+ * Modifies the subset of the configuration necessary
+ * Applies the latest cluster policies
+ * Updates the default router if one exists
+ * Updates the default registry if one exists
+ * Updates image streams and quickstarts
+
+## Usage
+ansible-playbook -i ~/ansible-inventory openshift-ansible/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
new file mode 100644
index 000000000..b06442366
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -0,0 +1,9 @@
+---
+- include: ../../../../common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+  vars:
+    g_etcd_group: "{{ 'etcd' }}"
+    g_masters_group: "{{ 'masters' }}"
+    g_nodes_group: "{{ 'nodes' }}"
+    g_lb_group: "{{ 'lb' }}"
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
new file mode 100644
index 000000000..ed4ab6d1b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+"""
+Pre-upgrade checks that must be run on a master before proceeding with upgrade.
+"""
+# This is a script not a python module:
+# pylint: disable=invalid-name
+
+# NOTE: This script should not require any python libs other than what is
+# in the standard library.
+
+__license__ = "ASL 2.0"
+
+import json
+import os
+import subprocess
+import re
+
+# The maximum length of container.ports.name
+ALLOWED_LENGTH = 15
+# The valid structure of container.ports.name
+ALLOWED_CHARS = re.compile('^[a-z0-9][a-z0-9\\-]*[a-z0-9]$')
+AT_LEAST_ONE_LETTER = re.compile('[a-z]')
+# look at OS_PATH for the full path. Default ot 'oc'
+OC_PATH = os.getenv('OC_PATH', 'oc')
+
+
+def validate(value):
+    """
+    validate verifies that value matches required conventions
+
+    Rules of container.ports.name validation:
+
+    * must be less that 16 chars
+    * at least one letter
+    * only a-z0-9-
+    * hyphens can not be leading or trailing or next to each other
+
+    :Parameters:
+       - `value`: Value to validate
+    """
+    if len(value) > ALLOWED_LENGTH:
+        return False
+
+    if '--' in value:
+        return False
+
+    # We search since it can be anywhere
+    if not AT_LEAST_ONE_LETTER.search(value):
+        return False
+
+    # We match because it must start at the beginning
+    if not ALLOWED_CHARS.match(value):
+        return False
+    return True
+
+
+def list_items(kind):
+    """
+    list_items returns a list of items from the api
+
+    :Parameters:
+       - `kind`: Kind of item to access
+    """
+    response = subprocess.check_output([OC_PATH, 'get', '--all-namespaces', '-o', 'json', kind])
+    items = json.loads(response)
+    return items.get("items", [])
+
+
+def get(obj, *paths):
+    """
+    Gets an object
+
+    :Parameters:
+       - `obj`: A dictionary structure
+       - `path`: All other non-keyword arguments
+    """
+    ret_obj = obj
+    for path in paths:
+        if ret_obj.get(path, None) is None:
+            return []
+        ret_obj = ret_obj[path]
+    return ret_obj
+
+
+# pylint: disable=too-many-arguments
+def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid):
+    """
+    Prints out results in human friendly way.
+
+    :Parameters:
+       - `namespace`: Namespace of the resource
+       - `kind`: Kind of the resource
+       - `item_name`: Name of the resource
+       - `container_name`: Name of the container. May be "" when kind=Service.
+       - `port_name`: Name of the port
+       - `valid`: True if the port is valid
+    """
+    if not valid:
+        if len(container_name) > 0:
+            print('%s/%s -n %s (Container="%s" Port="%s")' % (
+                kind, item_name, namespace, container_name, port_name))
+        else:
+            print('%s/%s -n %s (Port="%s")' % (
+                kind, item_name, namespace, port_name))
+
+
+def print_validation_header():
+    """
+    Prints the error header. Should run on the first error to avoid
+    overwhelming the user.
+    """
+    print """\
+At least one port name does not validate. Valid port names:
+
+    * must be less that 16 chars
+    * have at least one letter
+    * only a-z0-9-
+    * do not start or end with -
+    * Dashes may not be next to eachother ('--')
+"""
+
+
+def main():
+    """
+    main is the main entry point to this script
+    """
+    try:
+        # the comma at the end suppresses the newline
+        print "Checking for oc ...",
+        subprocess.check_output([OC_PATH, 'whoami'])
+        print "found"
+    except:
+        print(
+            'Unable to run "%s whoami"\n'
+            'Please ensure OpenShift is running, and "oc" is on your system '
+            'path.\n'
+            'You can override the path with the OC_PATH environment variable.'
+            % OC_PATH)
+        raise SystemExit(1)
+
+    # Where the magic happens
+    first_error = True
+    for kind, path in [
+            ('replicationcontrollers', ("spec", "template", "spec", "containers")),
+            ('pods', ("spec", "containers")),
+            ('deploymentconfigs', ("spec", "template", "spec", "containers"))]:
+        for item in list_items(kind):
+            namespace = item["metadata"]["namespace"]
+            item_name = item["metadata"]["name"]
+            for container in get(item, *path):
+                container_name = container["name"]
+                for port in get(container, "ports"):
+                    port_name = port.get("name", None)
+                    if not port_name:
+                        # Unnamed ports are OK
+                        continue
+                    valid = validate(port_name)
+                    if not valid and first_error:
+                        first_error = False
+                        print_validation_header()
+                    pretty_print_errors(
+                        namespace, kind, item_name,
+                        container_name, port_name, valid)
+
+    # Services follow a different flow
+    for item in list_items('services'):
+        namespace = item["metadata"]["namespace"]
+        item_name = item["metadata"]["name"]
+        for port in get(item, "spec", "ports"):
+            port_name = port.get("targetPort", None)
+            if isinstance(port_name, int) or port_name is None:
+                # Integer only or unnamed ports are OK
+                continue
+            valid = validate(port_name)
+            if not valid and first_error:
+                first_error = False
+                print_validation_header()
+            pretty_print_errors(
+                namespace, "services", item_name, "", port_name, valid)
+
+    # If we had at least 1 error then exit with 1
+    if not first_error:
+        raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()
+
diff --git a/playbooks/common/openshift-cluster/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
new file mode 100644
index 000000000..f90719cab
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
+
+yum_available=$(yum list available "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
+
+
+echo "---"
+echo "curr_version: ${yum_installed}" 
+echo "avail_version: ${yum_available}"
diff --git a/playbooks/common/openshift-cluster/upgrades/filter_plugins b/playbooks/common/openshift-cluster/upgrades/filter_plugins
new file mode 120000
index 000000000..b1213dedb
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/filter_plugins
@@ -0,0 +1 @@
+../../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
new file mode 100755
index 000000000..a6721bb92
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
@@ -0,0 +1,154 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vim: expandtab:tabstop=4:shiftwidth=4
+
+"""Ansible module for modifying OpenShift configs during an upgrade"""
+
+import os
+import yaml
+
+DOCUMENTATION = '''
+---
+module: openshift_upgrade_config
+short_description: OpenShift Upgrade Config
+author: Jason DeTiberus
+requirements: [ ]
+'''
+EXAMPLES = '''
+'''
+
+def modify_api_levels(level_list, remove, ensure, msg_prepend='',
+                      msg_append=''):
+    """ modify_api_levels """
+    changed = False
+    changes = []
+
+    if not isinstance(remove, list):
+        remove = []
+
+    if not isinstance(ensure, list):
+        ensure = []
+
+    if not isinstance(level_list, list):
+        new_list = []
+        changed = True
+        changes.append("%s created missing %s" % (msg_prepend, msg_append))
+    else:
+        new_list = level_list
+        for level in remove:
+            if level in new_list:
+                new_list.remove(level)
+                changed = True
+                changes.append("%s removed %s %s" % (msg_prepend, level, msg_append))
+
+    for level in ensure:
+        if level not in new_list:
+            new_list.append(level)
+            changed = True
+            changes.append("%s added %s %s" % (msg_prepend, level, msg_append))
+
+    return {'new_list': new_list, 'changed': changed, 'changes': changes}
+
+
+def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup):
+    """Main upgrade method for 3.0 to 3.1."""
+    changes = []
+
+    # Facts do not get transferred to the hosts where custom modules run,
+    # need to make some assumptions here.
+    master_config = os.path.join(config_base, 'master/master-config.yaml')
+
+    master_cfg_file = open(master_config, 'r')
+    config = yaml.safe_load(master_cfg_file.read())
+    master_cfg_file.close()
+
+
+    # Remove unsupported api versions and ensure supported api versions from
+    # master config
+    unsupported_levels = ['v1beta1', 'v1beta2', 'v1beta3']
+    supported_levels = ['v1']
+
+    result = modify_api_levels(config.get('apiLevels'), unsupported_levels,
+                               supported_levels, 'master-config.yaml:', 'from apiLevels')
+    if result['changed']:
+        config['apiLevels'] = result['new_list']
+        changes.append(result['changes'])
+
+    if 'kubernetesMasterConfig' in config and 'apiLevels' in config['kubernetesMasterConfig']:
+        config['kubernetesMasterConfig'].pop('apiLevels')
+        changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels')
+
+    # Add proxyClientInfo to master-config
+    if 'proxyClientInfo' not in config['kubernetesMasterConfig']:
+        config['kubernetesMasterConfig']['proxyClientInfo'] = {
+            'certFile': 'master.proxy-client.crt',
+            'keyFile': 'master.proxy-client.key'
+        }
+        changes.append("master-config.yaml: added proxyClientInfo")
+
+    if len(changes) > 0:
+        if backup:
+            # TODO: Check success:
+            ansible_module.backup_local(master_config)
+
+        # Write the modified config:
+        out_file = open(master_config, 'w')
+        out_file.write(yaml.safe_dump(config, default_flow_style=False))
+        out_file.close()
+
+    return changes
+
+
+def upgrade_master(ansible_module, config_base, from_version, to_version, backup):
+    """Upgrade entry point."""
+    if from_version == '3.0':
+        if to_version == '3.1':
+            return upgrade_master_3_0_to_3_1(ansible_module, config_base, backup)
+
+
+def main():
+    """ main """
+    # disabling pylint errors for global-variable-undefined and invalid-name
+    # for 'global module' usage, since it is required to use ansible_facts
+    # pylint: disable=global-variable-undefined, invalid-name,
+    # redefined-outer-name
+    global module
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            config_base=dict(required=True),
+            from_version=dict(required=True, choices=['3.0']),
+            to_version=dict(required=True, choices=['3.1']),
+            role=dict(required=True, choices=['master']),
+            backup=dict(required=False, default=True, type='bool')
+        ),
+        supports_check_mode=True,
+    )
+
+    from_version = module.params['from_version']
+    to_version = module.params['to_version']
+    role = module.params['role']
+    backup = module.params['backup']
+    config_base = module.params['config_base']
+
+    try:
+        changes = []
+        if role == 'master':
+            changes = upgrade_master(module, config_base, from_version,
+                                     to_version, backup)
+
+        changed = len(changes) > 0
+        return module.exit_json(changed=changed, changes=changes)
+
+    # ignore broad-except error to avoid stack trace to ansible user
+    # pylint: disable=broad-except
+    except Exception, e:
+        return module.fail_json(msg=str(e))
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+# import module snippets
+from ansible.module_utils.basic import *
+
+if __name__ == '__main__':
+    main()
diff --git a/playbooks/common/openshift-cluster/upgrades/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/lookup_plugins
new file mode 120000
index 000000000..aff753026
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/lookup_plugins
@@ -0,0 +1 @@
+../../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/roles b/playbooks/common/openshift-cluster/upgrades/roles
new file mode 120000
index 000000000..4bdbcbad3
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/roles
@@ -0,0 +1 @@
+../../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins
new file mode 120000
index 000000000..27ddaa18b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/filter_plugins
@@ -0,0 +1 @@
+../../../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library
new file mode 120000
index 000000000..53bed9684
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/library
@@ -0,0 +1 @@
+../library
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins
new file mode 120000
index 000000000..cf407f69b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/lookup_plugins
@@ -0,0 +1 @@
+../../../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles
new file mode 120000
index 000000000..6bc1a7aef
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/roles
@@ -0,0 +1 @@
+../../../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
new file mode 100644
index 000000000..9f7e49b93
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_minor/upgrade.yml
@@ -0,0 +1,112 @@
+---
+- name: Evaluate groups
+  include: ../../evaluate_groups.yml
+
+- name: Re-Run cluster configuration to apply latest configuration changes
+  include: ../../config.yml
+
+- name: Upgrade masters
+  hosts: oo_masters_to_config
+  vars:
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
+  tasks:
+    - name: Upgrade master packages
+      yum: pkg={{ openshift.common.service_type }}-master{{ openshift_version }} state=latest
+    - name: Restart master services
+      service: name="{{ openshift.common.service_type}}-master" state=restarted
+
+- name: Upgrade nodes
+  hosts: oo_nodes_to_config
+  vars:
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
+  tasks:
+    - name: Upgrade node packages
+      yum: pkg={{ openshift.common.service_type }}-node{{ openshift_version }} state=latest
+    - name: Restart node services
+      service: name="{{ openshift.common.service_type }}-node" state=restarted
+
+- name: Determine new master version
+  hosts: oo_first_master
+  tasks:
+    - name: Determine new version
+      command: >
+        rpm -q --queryformat '%{version}' {{ openshift.common.service_type }}-master
+      register: _new_version
+
+- name: Ensure AOS 3.0.2 or Origin 1.0.6
+  hosts: oo_first_master
+  tasks:
+    fail: This playbook requires Origin 1.0.6 or Atomic OpenShift 3.0.2 or later
+    when: _new_version.stdout | version_compare('1.0.6','<') or ( _new_version.stdout | version_compare('3.0','>=' and _new_version.stdout | version_compare('3.0.2','<') )
+
+- name: Update cluster policy
+  hosts: oo_first_master
+  tasks:
+    - name: oadm policy reconcile-cluster-roles --confirm
+      command: >
+        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+        policy reconcile-cluster-roles --confirm
+
+- name: Upgrade default router
+  hosts: oo_first_master
+  vars:
+    - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + _new_version.stdout ) }}"
+    - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
+  tasks:
+    - name: Check for default router
+      command: >
+        {{ oc_cmd }} get -n default dc/router
+      register: _default_router
+      failed_when: false
+      changed_when: false
+    - name: Check for allowHostNetwork and allowHostPorts
+      when: _default_router.rc == 0
+      shell: >
+        {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork
+      register: _scc
+    - name: Grant allowHostNetwork and allowHostPorts
+      when:
+        - _default_router.rc == 0
+        - "'false' in _scc.stdout"
+      command: >
+        {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
+    - name: Update deployment config to 1.0.4/3.0.1 spec
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p
+        '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
+    - name: Switch to hostNetwork=true
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
+    - name: Update router image to current version
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p
+        '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
+
+- name: Upgrade default
+  hosts: oo_first_master
+  vars:
+    - registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + _new_version.stdout  ) }}"
+    - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
+  tasks:
+    - name: Check for default registry
+      command: >
+          {{ oc_cmd }} get -n default dc/docker-registry
+      register: _default_registry
+      failed_when: false
+      changed_when: false
+    - name: Update registry image to current version
+      when: _default_registry.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/docker-registry -p
+        '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
+
+- name: Update image streams and templates
+  hosts: oo_first_master
+  vars:
+    openshift_examples_import_command: "update"
+    openshift_deployment_type: "{{ deployment_type }}"
+  roles:
+    - openshift_examples
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins
new file mode 120000
index 000000000..27ddaa18b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/filter_plugins
@@ -0,0 +1 @@
+../../../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library
new file mode 120000
index 000000000..53bed9684
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/library
@@ -0,0 +1 @@
+../library
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins
new file mode 120000
index 000000000..cf407f69b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/lookup_plugins
@@ -0,0 +1 @@
+../../../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles
new file mode 120000
index 000000000..6bc1a7aef
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/roles
@@ -0,0 +1 @@
+../../../../../roles
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
new file mode 100644
index 000000000..dd6979ab7
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -0,0 +1,407 @@
+---
+- name: Evaluate host groups
+  include: ../../evaluate_groups.yml
+
+- name: Load openshift_facts from the environment
+  hosts: oo_masters_to_config oo_nodes_to_config oo_etcd_to_config oo_lb_to_config
+  roles:
+  - openshift_facts
+
+- name: Verify upgrade can proceed
+  hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  gather_facts: no
+  tasks:
+  # Pacemaker is currently the only supported upgrade path for multiple masters
+  - fail:
+      msg: "openshift_master_cluster_method must be set to 'pacemaker'"
+    when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker"))
+  - fail:
+      msg: >
+        This upgrade is only supported for origin and openshift-enterprise
+        deployment types
+    when: deployment_type not in ['origin','openshift-enterprise']
+  - fail:
+      msg: >
+        openshift_pkg_version is {{ openshift_pkg_version }} which is not a
+        valid version for a 3.1 upgrade
+    when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare('3.0.2.900','<')
+
+  # If this script errors out ansible will show the default stdout/stderr
+  # which contains details for the user:
+  - script: ../files/pre-upgrade-check
+
+- name: Evaluate etcd_hosts_to_backup
+  hosts: localhost
+  tasks:
+  - name: Evaluate etcd_hosts_to_backup
+    add_host:
+      name: "{{ item }}"
+      groups: etcd_hosts_to_backup
+    with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master
+
+- name: Backup etcd
+  hosts: etcd_hosts_to_backup
+  vars:
+    embedded_etcd: "{{ openshift.master.embedded_etcd }}"
+    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+  roles:
+  - openshift_facts
+  tasks:
+  - openshift_facts:
+      role: etcd
+      local_facts: {}
+    when: "'etcd' not in openshift"
+
+  - stat: path=/var/lib/openshift
+    register: var_lib_openshift
+
+  - stat: path=/var/lib/origin
+    register: var_lib_origin
+
+  - name: Create origin symlink if necessary
+    file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
+    when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
+
+  - name: Check available disk space for etcd backup
+    # We assume to be using the data dir for all backups.
+    shell: >
+      df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
+    register: avail_disk
+
+  - name: Check current embedded etcd disk usage
+    shell: >
+      du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
+    register: etcd_disk_usage
+    when: embedded_etcd | bool
+
+  - name: Abort if insufficient disk space for etcd backup
+    fail:
+      msg: >
+        {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
+        {{ avail_disk.stdout }} Kb available.
+    when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
+
+  - name: Install etcd (for etcdctl)
+    yum:
+      pkg: etcd
+      state: latest
+
+  - name: Generate etcd backup
+    command: >
+      etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }}
+      --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
+
+  - name: Display location of etcd backup
+    debug:
+      msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
+
+
+- name: Update deployment type
+  hosts: OSEv3
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: common
+      local_facts:
+        deployment_type: "{{ deployment_type }}"
+
+
+- name: Perform upgrade version checking
+  hosts: masters[0]
+  tasks:
+  - name: Clean yum cache
+    command: yum clean all
+
+  - name: Determine available versions
+    script: ../files/versions.sh {{ openshift.common.service_type }} openshift
+    register: g_versions_result
+
+  - set_fact:
+      g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}"
+
+  - set_fact:
+      g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
+
+  - fail:
+      msg: This playbook requires Origin 1.0.6 or later
+    when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
+
+  - fail:
+      msg: Atomic OpenShift 3.1 packages not found
+    when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
+
+- name: Upgrade masters
+  hosts: masters
+  vars:
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
+  tasks:
+    - name: Upgrade to latest available kernel
+      yum:
+        pkg: kernel
+        state: latest
+
+    - name: Upgrade master packages
+      command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
+
+    - name: Ensure python-yaml present for config upgrade
+      yum:
+        pkg: PyYAML
+        state: installed
+
+    - name: Upgrade master configuration
+      openshift_upgrade_config:
+        from_version: '3.0'
+        to_version: '3.1'
+        role: master
+        config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}"
+
+    - set_fact:
+        master_certs_missing: True
+        master_cert_subdir: master-{{ openshift.common.hostname }}
+        master_cert_config_dir: "{{ openshift.common.config_base }}/master"
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: False
+
+- name: Generate missing master certificates
+  hosts: masters[0]
+  vars:
+    master_hostnames: "{{ hostvars
+                          | oo_select_keys(groups.masters)
+                          | oo_collect('openshift.common.all_hostnames')
+                          | oo_flatten | unique }}"
+    master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs"
+    masters_needing_certs: "{{ hostvars
+                               | oo_select_keys(groups.masters)
+                               | difference([groups.masters.0]) }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_deployment_type: "{{ deployment_type }}"
+  roles:
+  - openshift_master_certificates
+  post_tasks:
+  - name: Remove generated etcd client certs when using external etcd
+    file:
+      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+      state: absent
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    with_nested:
+    - masters_needing_certs
+    - - master.etcd-client.crt
+      - master.etcd-client.key
+
+  - name: Create a tarball of the master certs
+    command: >
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
+    with_items: masters_needing_certs
+
+  - name: Retrieve the master cert tarball from the master
+    fetch:
+      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: masters_needing_certs
+
+- name: Sync certs and restart masters post configuration change
+  hosts: masters
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  tasks:
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
+      dest: "{{ master_cert_config_dir }}"
+    when: inventory_hostname != groups.masters.0
+
+  - name: Restart master services
+    service: name="{{ openshift.common.service_type}}-master" state=restarted
+    when: not openshift_master_ha | bool
+
+- name: Destroy cluster
+  hosts: masters[0]
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+    openshift_deployment_type: "{{ deployment_type }}"
+  pre_tasks:
+  - name: Check for configured cluster
+    stat:
+      path: /etc/corosync/corosync.conf
+    register: corosync_conf
+    when: openshift_master_ha | bool
+  - name: Destroy cluster
+    command: pcs cluster destroy --all
+    when: openshift_master_ha | bool and corosync_conf.stat.exists == true
+
+- name: Start pcsd on masters
+  hosts: masters
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  tasks:
+  - name: Start pcsd
+    service: name=pcsd enabled=yes state=started
+    when: openshift_master_ha | bool
+
+- name: Re-create cluster
+  hosts: masters[0]
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+    openshift_deployment_type: "{{ deployment_type }}"
+    omc_cluster_hosts: "{{ groups.masters | join(' ') }}"
+  roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - file: name={{ g_master_mktemp.stdout }} state=absent
+    changed_when: False
+
+
+- name: Upgrade nodes
+  hosts: nodes
+  vars:
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
+  roles:
+  - openshift_facts
+  tasks:
+    - name: Upgrade node packages
+      command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }}
+    - name: Restart node services
+      service: name="{{ openshift.common.service_type }}-node" state=restarted
+
+- name: Update cluster policy and policy bindings
+  hosts: masters[0]
+  vars:
+    origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}"
+    ent_reconcile_bindings: true
+  tasks:
+    - name: oadm policy reconcile-cluster-roles --confirm
+      command: >
+        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+        policy reconcile-cluster-roles --confirm
+
+    - name: oadm policy reconcile-cluster-role-bindings --confirm
+      command: >
+        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+        policy reconcile-cluster-role-bindings
+        --exclude-groups=system:authenticated
+        --exclude-groups=system:unauthenticated
+        --exclude-users=system:anonymous
+        --additive-only=true --confirm
+      when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
+
+
+- name: Restart masters post reconcile
+  hosts: masters
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  tasks:
+    - name: Restart master services
+      service: name="{{ openshift.common.service_type}}-master" state=restarted
+      when: not openshift_master_ha | bool
+
+- name: Restart cluster post reconcile
+  hosts: masters[0]
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  tasks:
+    - name: Restart master cluster
+      command: pcs resource restart master
+      when: openshift_master_ha | bool
+    - name: Wait for the clustered master service to be available
+      wait_for:
+        host: "{{ openshift_master_cluster_vip }}"
+        port: 8443
+        state: started
+        timeout: 180
+        delay: 90
+      when: openshift_master_ha | bool
+
+- name: Upgrade default router and registry
+  hosts: masters[0]
+  vars:
+    - registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + g_new_version  ) }}"
+    - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
+    - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
+  tasks:
+    - name: Check for default router
+      command: >
+        {{ oc_cmd }} get -n default dc/router
+      register: _default_router
+      failed_when: false
+      changed_when: false
+    - name: Check for allowHostNetwork and allowHostPorts
+      when: _default_router.rc == 0
+      shell: >
+        {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork
+      register: _scc
+    - name: Grant allowHostNetwork and allowHostPorts
+      when:
+        - _default_router.rc == 0
+        - "'false' in _scc.stdout"
+      command: >
+        {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
+    - name: Update deployment config to 1.0.4/3.0.1 spec
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p
+        '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
+    - name: Switch to hostNetwork=true
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
+    - name: Update router image to current version
+      when: _default_router.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/router -p
+        '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
+
+    - name: Check for default registry
+      command: >
+          {{ oc_cmd }} get -n default dc/docker-registry
+      register: _default_registry
+      failed_when: false
+      changed_when: false
+    - name: Update registry image to current version
+      when: _default_registry.rc == 0
+      command: >
+        {{ oc_cmd }} patch dc/docker-registry -p
+        '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
+
+- name: Update image streams and templates
+  hosts: masters[0]
+  vars:
+    openshift_examples_import_command: "update"
+    openshift_deployment_type: "{{ deployment_type }}"
+  roles:
+    - openshift_examples
+
+- name: Ensure master services enabled
+  hosts: masters
+  vars:
+    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+  tasks:
+  - name: Enable master services
+    service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes
+    when: not openshift_master_ha | bool
+
+- name: Ensure node services enabled
+  hosts: nodes
+  tasks:
+    - name: Restart node services
+      service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
+
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 952960652..ed23ada88 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -13,6 +13,8 @@
           hostname: "{{ openshift_hostname | default(None) }}"
           public_hostname: "{{ openshift_public_hostname | default(None) }}"
           deployment_type: "{{ openshift_deployment_type }}"
+      - role: etcd
+        local_facts: {}
   - name: Check status of etcd certificates
     stat:
       path: "{{ item }}"
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index b1da85d5d..1b3fba3aa 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -51,6 +51,9 @@
           console_url: "{{ openshift_master_console_url | default(None) }}"
           console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
           public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
+      - role: etcd
+        local_facts: {}
+        when: openshift.master.embedded_etcd | bool
   - name: Check status of external etcd certificatees
     stat:
       path: "{{ openshift.common.config_base }}/master/{{ item }}"
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index c108cd422..2e1075aca 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -528,7 +528,6 @@ def set_aggregate_facts(facts):
             first_svc_ip = str(IPNetwork(facts['master']['portal_net'])[1])
             all_hostnames.add(first_svc_ip)
             internal_hostnames.add(first_svc_ip)
-            _add_etcd_data_dir_fact(facts)
 
         facts['common']['all_hostnames'] = list(all_hostnames)
         facts['common']['internal_hostnames'] = list(internal_hostnames)
@@ -536,7 +535,7 @@ def set_aggregate_facts(facts):
     return facts
 
 
-def _add_etcd_data_dir_fact(facts):
+def set_etcd_facts_if_unset(facts):
     """
     If using embedded etcd, loads the data directory from master-config.yaml.
 
@@ -544,38 +543,39 @@ def _add_etcd_data_dir_fact(facts):
 
     If anything goes wrong parsing these, the fact will not be set.
     """
-    if facts['master']['embedded_etcd']:
-        try:
-            # Parse master config to find actual etcd data dir:
-            master_cfg_path = os.path.join(facts['common']['config_base'],
-                                           'master/master-config.yaml')
-            master_cfg_f = open(master_cfg_path, 'r')
-            config = yaml.safe_load(master_cfg_f.read())
-            master_cfg_f.close()
-
-            facts['master']['etcd_data_dir'] = \
-                config['etcdConfig']['storageDirectory']
-        # We don't want exceptions bubbling up here:
-        # pylint: disable=broad-except
-        except Exception:
-            pass
-    else:
-        # Read ETCD_DATA_DIR from /etc/etcd/etcd.conf:
-        try:
-            # Add a fake section for parsing:
-            ini_str = '[root]\n' + open('/etc/etcd/etcd.conf', 'r').read()
-            ini_fp = StringIO.StringIO(ini_str)
-            config = ConfigParser.RawConfigParser()
-            config.readfp(ini_fp)
-            etcd_data_dir = config.get('root', 'ETCD_DATA_DIR')
-            if etcd_data_dir.startswith('"') and etcd_data_dir.endswith('"'):
-                etcd_data_dir = etcd_data_dir[1:-1]
-            facts['master']['etcd_data_dir'] = etcd_data_dir
-        # We don't want exceptions bubbling up here:
-        # pylint: disable=broad-except
-        except Exception:
-            pass
-
+    if 'etcd' in facts:
+        if 'master' in facts and facts['master']['embedded_etcd']:
+            try:
+                # Parse master config to find actual etcd data dir:
+                master_cfg_path = os.path.join(facts['common']['config_base'],
+                                               'master/master-config.yaml')
+                master_cfg_f = open(master_cfg_path, 'r')
+                config = yaml.safe_load(master_cfg_f.read())
+                master_cfg_f.close()
+
+                facts['etcd']['etcd_data_dir'] = \
+                    config['etcdConfig']['storageDirectory']
+            # We don't want exceptions bubbling up here:
+            # pylint: disable=broad-except
+            except Exception:
+                pass
+        else:
+            # Read ETCD_DATA_DIR from /etc/etcd/etcd.conf:
+            try:
+                # Add a fake section for parsing:
+                ini_str = '[root]\n' + open('/etc/etcd/etcd.conf', 'r').read()
+                ini_fp = StringIO.StringIO(ini_str)
+                config = ConfigParser.RawConfigParser()
+                config.readfp(ini_fp)
+                etcd_data_dir = config.get('root', 'ETCD_DATA_DIR')
+                if etcd_data_dir.startswith('"') and etcd_data_dir.endswith('"'):
+                    etcd_data_dir = etcd_data_dir[1:-1]
+                facts['etcd']['etcd_data_dir'] = etcd_data_dir
+            # We don't want exceptions bubbling up here:
+            # pylint: disable=broad-except
+            except Exception:
+                pass
+    return facts
 
 def set_deployment_facts_if_unset(facts):
     """ Set Facts that vary based on deployment_type. This currently
@@ -939,7 +939,7 @@ class OpenShiftFacts(object):
         Raises:
             OpenShiftFactsUnsupportedRoleError:
     """
-    known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'dns']
+    known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'dns', 'etcd']
 
     def __init__(self, role, filename, local_facts):
         self.changed = False
@@ -982,6 +982,7 @@ class OpenShiftFacts(object):
         facts = set_deployment_facts_if_unset(facts)
         facts = set_version_facts_if_unset(facts)
         facts = set_aggregate_facts(facts)
+        facts = set_etcd_facts_if_unset(facts)
         return dict(openshift=facts)
 
     def get_defaults(self, roles):
diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py
index 4c55002fb..6cdc19f20 100644
--- a/utils/src/ooinstall/cli_installer.py
+++ b/utils/src/ooinstall/cli_installer.py
@@ -177,7 +177,8 @@ Notes:
                                              h.public_ip,
                                              h.hostname,
                                              h.public_hostname]))
-        output = "%s\n%s" % (output, ",".join([h.ip,
+        output = "%s\n%s" % (output, ",".join([h.connect_to,
+                             h.ip,
                              h.public_ip,
                              h.hostname,
                              h.public_hostname]))
@@ -493,7 +494,7 @@ def upgrade(ctx):
     verbose = ctx.obj['verbose']
 
     if len(oo_cfg.hosts) == 0:
-        click.echo("No hosts defined in: %s" % oo_cfg['configuration'])
+        click.echo("No hosts defined in: %s" % oo_cfg.config_path)
         sys.exit(1)
 
     # Update config to reflect the version we're targetting, we'll write
diff --git a/utils/src/ooinstall/oo_config.py b/utils/src/ooinstall/oo_config.py
index cf51bb404..9c97e6e93 100644
--- a/utils/src/ooinstall/oo_config.py
+++ b/utils/src/ooinstall/oo_config.py
@@ -116,6 +116,9 @@ class OOConfig(object):
 
     def _upgrade_legacy_config(self):
         new_hosts = []
+        remove_settings = ['validated_facts', 'Description', 'Name',
+            'Subscription', 'Vendor', 'Version', 'masters', 'nodes']
+
         if 'validated_facts' in self.settings:
             for key, value in self.settings['validated_facts'].iteritems():
                 value['connect_to'] = key
@@ -126,10 +129,9 @@ class OOConfig(object):
                 new_hosts.append(value)
         self.settings['hosts'] = new_hosts
 
-        remove_settings = ['validated_facts', 'Description', 'Name',
-            'Subscription', 'Vendor', 'Version', 'masters', 'nodes']
         for s in remove_settings:
-            del self.settings[s]
+            if s in self.settings:
+                del self.settings[s]
 
         # A legacy config implies openshift-enterprise 3.0:
         self.settings['variant'] = 'openshift-enterprise'
diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py
index 489a0f7c1..e4c808e85 100644
--- a/utils/src/ooinstall/openshift_ansible.py
+++ b/utils/src/ooinstall/openshift_ansible.py
@@ -164,8 +164,10 @@ def run_uninstall_playbook(verbose=False):
 
 
 def run_upgrade_playbook(verbose=False):
+    # TODO: do not hardcode the upgrade playbook, add ability to select the
+    # right playbook depending on the type of upgrade.
     playbook = os.path.join(CFG.settings['ansible_playbook_directory'],
-        'playbooks/adhoc/upgrades/upgrade.yml')
+        'playbooks/byo/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml')
     # TODO: Upgrade inventory for upgrade?
     inventory_file = generate_inventory(CFG.hosts)
     facts_env = os.environ.copy()
-- 
cgit v1.2.3


From 32596e5b6440ca7e1cc53aba36c0b4c50fa528f1 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Wed, 11 Nov 2015 14:07:39 -0500
Subject: General cleanup of v3_0_to_v3_1/upgrade.yml

- Reorder to push all non-changing checks first
- Remove multiple plays where possible
- Make formatting more consistent
- Add additional comments to break up the different stages of the upgrade.
- Use group names more consistently
- Add package version checking to nodes
---
 .../upgrades/v3_0_to_v3_1/upgrade.yml              | 435 +++++++++++----------
 1 file changed, 225 insertions(+), 210 deletions(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index dd6979ab7..1cd7327cb 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -1,27 +1,45 @@
 ---
+###############################################################################
+# Evaluate host groups and gather facts
+###############################################################################
 - name: Evaluate host groups
   include: ../../evaluate_groups.yml
 
-- name: Load openshift_facts from the environment
-  hosts: oo_masters_to_config oo_nodes_to_config oo_etcd_to_config oo_lb_to_config
+- name: Load openshift_facts
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config:oo_lb_to_config
   roles:
   - openshift_facts
 
+- name: Evaluate etcd_hosts_to_backup
+  hosts: localhost
+  tasks:
+  - name: Evaluate etcd_hosts_to_backup
+    add_host:
+      name: "{{ item }}"
+      groups: etcd_hosts_to_backup
+    with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master
+
+
+###############################################################################
+# Pre-upgrade checks
+###############################################################################
 - name: Verify upgrade can proceed
   hosts: oo_first_master
   vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
   gather_facts: no
   tasks:
   # Pacemaker is currently the only supported upgrade path for multiple masters
   - fail:
       msg: "openshift_master_cluster_method must be set to 'pacemaker'"
     when: openshift_master_ha | bool and ((openshift_master_cluster_method is not defined) or (openshift_master_cluster_method is defined and openshift_master_cluster_method != "pacemaker"))
+
   - fail:
       msg: >
         This upgrade is only supported for origin and openshift-enterprise
         deployment types
     when: deployment_type not in ['origin','openshift-enterprise']
+
   - fail:
       msg: >
         openshift_pkg_version is {{ openshift_pkg_version }} which is not a
@@ -32,15 +50,38 @@
   # which contains details for the user:
   - script: ../files/pre-upgrade-check
 
-- name: Evaluate etcd_hosts_to_backup
-  hosts: localhost
+
+- name: Verify upgrade can proceed
+  hosts: masters:nodes
   tasks:
-  - name: Evaluate etcd_hosts_to_backup
-    add_host:
-      name: "{{ item }}"
-      groups: etcd_hosts_to_backup
-    with_items: groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master
+  - name: Clean yum cache
+    command: yum clean all
+
+  - set_fact:
+      g_new_service_name: "{{ 'origin' if deployment_type =='origin' else 'atomic-openshift' }}"
+
+  - name: Determine available versions
+    script: ../files/versions.sh {{ g_new_service_name }} openshift
+    register: g_versions_result
+
+  - set_fact:
+      g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}"
+
+  - set_fact:
+      g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
+
+  - fail:
+      msg: This playbook requires Origin 1.0.6 or later
+    when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
 
+  - fail:
+      msg: Atomic OpenShift 3.1 packages not found
+    when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
+
+
+###############################################################################
+# Backup etcd
+###############################################################################
 - name: Backup etcd
   hosts: etcd_hosts_to_backup
   vars:
@@ -64,15 +105,15 @@
     file: src=/var/lib/openshift/ dest=/var/lib/origin state=link
     when: var_lib_openshift.stat.exists == True and var_lib_origin.stat.exists == False
 
+  # TODO: replace shell module with command and update later checks
+  # We assume to be using the data dir for all backups.
   - name: Check available disk space for etcd backup
-    # We assume to be using the data dir for all backups.
-    shell: >
-      df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
+    shell: df --output=avail -k {{ openshift.common.data_dir }} | tail -n 1
     register: avail_disk
 
+  # TODO: replace shell module with command and update later checks
   - name: Check current embedded etcd disk usage
-    shell: >
-      du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
+    shell: du -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
     register: etcd_disk_usage
     when: embedded_etcd | bool
 
@@ -98,6 +139,18 @@
       msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
 
 
+###############################################################################
+# Upgrade Masters
+###############################################################################
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: False
+
 - name: Update deployment type
   hosts: OSEv3
   roles:
@@ -108,81 +161,48 @@
       local_facts:
         deployment_type: "{{ deployment_type }}"
 
-
-- name: Perform upgrade version checking
-  hosts: masters[0]
+- name: Upgrade master packages and configuration
+  hosts: oo_masters_to_config
+  vars:
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
   tasks:
-  - name: Clean yum cache
-    command: yum clean all
-
-  - name: Determine available versions
-    script: ../files/versions.sh {{ openshift.common.service_type }} openshift
-    register: g_versions_result
-
-  - set_fact:
-      g_aos_versions: "{{ g_versions_result.stdout | from_yaml }}"
+  - name: Upgrade to latest available kernel
+    yum:
+      pkg: kernel
+      state: latest
 
-  - set_fact:
-      g_new_version: "{{ g_aos_versions.curr_version.split('-', 1).0 if g_aos_versions.avail_version is none else g_aos_versions.avail_version.split('-', 1).0 }}"
+  - name: Upgrade master packages
+    command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
 
-  - fail:
-      msg: This playbook requires Origin 1.0.6 or later
-    when: deployment_type == 'origin' and g_aos_versions.curr_version | version_compare('1.0.6','<')
+  - name: Ensure python-yaml present for config upgrade
+    yum:
+      pkg: PyYAML
+      state: installed
 
-  - fail:
-      msg: Atomic OpenShift 3.1 packages not found
-    when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
+  - name: Upgrade master configuration
+    openshift_upgrade_config:
+      from_version: '3.0'
+      to_version: '3.1'
+      role: master
+      config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}"
 
-- name: Upgrade masters
-  hosts: masters
-  vars:
-    openshift_version: "{{ openshift_pkg_version | default('') }}"
-  tasks:
-    - name: Upgrade to latest available kernel
-      yum:
-        pkg: kernel
-        state: latest
-
-    - name: Upgrade master packages
-      command: yum update -y {{ openshift.common.service_type }}-master{{ openshift_version }}
-
-    - name: Ensure python-yaml present for config upgrade
-      yum:
-        pkg: PyYAML
-        state: installed
-
-    - name: Upgrade master configuration
-      openshift_upgrade_config:
-        from_version: '3.0'
-        to_version: '3.1'
-        role: master
-        config_base: "{{ hostvars[inventory_hostname].openshift.common.config_base }}"
-
-    - set_fact:
-        master_certs_missing: True
-        master_cert_subdir: master-{{ openshift.common.hostname }}
-        master_cert_config_dir: "{{ openshift.common.config_base }}/master"
+  - set_fact:
+      master_certs_missing: True
+      master_cert_subdir: master-{{ openshift.common.hostname }}
+      master_cert_config_dir: "{{ openshift.common.config_base }}/master"
 
-- name: Create temp directory for syncing certs
-  hosts: localhost
-  gather_facts: no
-  tasks:
-  - name: Create local temp directory for syncing certs
-    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
-    register: g_master_mktemp
-    changed_when: False
 
 - name: Generate missing master certificates
-  hosts: masters[0]
+  hosts: oo_first_master
   vars:
     master_hostnames: "{{ hostvars
-                          | oo_select_keys(groups.masters)
+                          | oo_select_keys(groups.oo_masters_to_config)
                           | oo_collect('openshift.common.all_hostnames')
                           | oo_flatten | unique }}"
     master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs"
     masters_needing_certs: "{{ hostvars
-                               | oo_select_keys(groups.masters)
-                               | difference([groups.masters.0]) }}"
+                               | oo_select_keys(groups.oo_masters_to_config)
+                               | difference([groups.oo_first_master.0]) }}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     openshift_deployment_type: "{{ deployment_type }}"
   roles:
@@ -213,56 +233,55 @@
       validate_checksum: yes
     with_items: masters_needing_certs
 
-- name: Sync certs and restart masters post configuration change
-  hosts: masters
+
+- name: Sync generated certs, update service config and restart master services
+  hosts: oo_masters_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    openshift_deployment_type: "{{ deployment_type }}"
   tasks:
   - name: Unarchive the tarball on the master
     unarchive:
       src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
       dest: "{{ master_cert_config_dir }}"
-    when: inventory_hostname != groups.masters.0
+    when: inventory_hostname != groups.oo_first_master.0
 
-  - name: Restart master services
+  - name: Restart master service
     service: name="{{ openshift.common.service_type}}-master" state=restarted
     when: not openshift_master_ha | bool
 
-- name: Destroy cluster
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-    openshift_deployment_type: "{{ deployment_type }}"
-  pre_tasks:
+  - name: Ensure the master service is enabled
+    service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes
+    when: not openshift_master_ha | bool
+
   - name: Check for configured cluster
     stat:
       path: /etc/corosync/corosync.conf
     register: corosync_conf
     when: openshift_master_ha | bool
+
   - name: Destroy cluster
     command: pcs cluster destroy --all
     when: openshift_master_ha | bool and corosync_conf.stat.exists == true
+    run_once: true
 
-- name: Start pcsd on masters
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
   - name: Start pcsd
     service: name=pcsd enabled=yes state=started
     when: openshift_master_ha | bool
 
+
 - name: Re-create cluster
-  hosts: masters[0]
+  hosts: oo_first_master
   vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
     openshift_deployment_type: "{{ deployment_type }}"
-    omc_cluster_hosts: "{{ groups.masters | join(' ') }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ') }}"
   roles:
   - role: openshift_master_cluster
     when: openshift_master_ha | bool
 
+
 - name: Delete temporary directory on localhost
   hosts: localhost
   gather_facts: no
@@ -271,137 +290,133 @@
     changed_when: False
 
 
+###############################################################################
+# Upgrade Nodes
+###############################################################################
 - name: Upgrade nodes
-  hosts: nodes
+  hosts: oo_nodes_to_config
   vars:
     openshift_version: "{{ openshift_pkg_version | default('') }}"
   roles:
   - openshift_facts
   tasks:
-    - name: Upgrade node packages
-      command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }}
-    - name: Restart node services
-      service: name="{{ openshift.common.service_type }}-node" state=restarted
+  - name: Upgrade node packages
+    command: yum update -y {{ openshift.common.service_type }}-node{{ openshift_version }}
 
-- name: Update cluster policy and policy bindings
-  hosts: masters[0]
+  - name: Restart node service
+    service: name="{{ openshift.common.service_type }}-node" state=restarted
+
+  - name: Ensure node service enabled
+    service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
+
+
+###############################################################################
+# Post upgrade - Reconcile Cluster Roles and Cluster Role Bindings
+###############################################################################
+- name: Reconcile Cluster Roles and Cluster Role Bindings
+  hosts: oo_masters_to_config
   vars:
     origin_reconcile_bindings: "{{ deployment_type == 'origin' and g_new_version | version_compare('1.0.6', '>') }}"
     ent_reconcile_bindings: true
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
   tasks:
-    - name: oadm policy reconcile-cluster-roles --confirm
-      command: >
-        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
-        policy reconcile-cluster-roles --confirm
-
-    - name: oadm policy reconcile-cluster-role-bindings --confirm
-      command: >
-        {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
-        policy reconcile-cluster-role-bindings
-        --exclude-groups=system:authenticated
-        --exclude-groups=system:unauthenticated
-        --exclude-users=system:anonymous
-        --additive-only=true --confirm
-      when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
-
-
-- name: Restart masters post reconcile
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-    - name: Restart master services
-      service: name="{{ openshift.common.service_type}}-master" state=restarted
-      when: not openshift_master_ha | bool
+  - name: Reconcile Cluster Roles
+    command: >
+      {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+      policy reconcile-cluster-roles --confirm
+    run_once: true
 
-- name: Restart cluster post reconcile
-  hosts: masters[0]
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-    - name: Restart master cluster
-      command: pcs resource restart master
-      when: openshift_master_ha | bool
-    - name: Wait for the clustered master service to be available
-      wait_for:
-        host: "{{ openshift_master_cluster_vip }}"
-        port: 8443
-        state: started
-        timeout: 180
-        delay: 90
-      when: openshift_master_ha | bool
-
-- name: Upgrade default router and registry
-  hosts: masters[0]
-  vars:
-    - registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + g_new_version  ) }}"
-    - router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
-    - oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
-  tasks:
-    - name: Check for default router
-      command: >
-        {{ oc_cmd }} get -n default dc/router
-      register: _default_router
-      failed_when: false
-      changed_when: false
-    - name: Check for allowHostNetwork and allowHostPorts
-      when: _default_router.rc == 0
-      shell: >
-        {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork
-      register: _scc
-    - name: Grant allowHostNetwork and allowHostPorts
-      when:
-        - _default_router.rc == 0
-        - "'false' in _scc.stdout"
-      command: >
-        {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
-    - name: Update deployment config to 1.0.4/3.0.1 spec
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p
-        '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
-    - name: Switch to hostNetwork=true
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
-    - name: Update router image to current version
-      when: _default_router.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/router -p
-        '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
-
-    - name: Check for default registry
-      command: >
-          {{ oc_cmd }} get -n default dc/docker-registry
-      register: _default_registry
-      failed_when: false
-      changed_when: false
-    - name: Update registry image to current version
-      when: _default_registry.rc == 0
-      command: >
-        {{ oc_cmd }} patch dc/docker-registry -p
-        '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
-
-- name: Update image streams and templates
-  hosts: masters[0]
+  - name: Reconcile Cluster Role Bindings
+    command: >
+      {{ openshift.common.admin_binary}} --config={{ openshift.common.config_base }}/master/admin.kubeconfig
+      policy reconcile-cluster-role-bindings
+      --exclude-groups=system:authenticated
+      --exclude-groups=system:unauthenticated
+      --exclude-users=system:anonymous
+      --additive-only=true --confirm
+    when: origin_reconcile_bindings | bool or ent_reconcile_bindings | bool
+    run_once: true
+
+  - name: Restart master services
+    service: name="{{ openshift.common.service_type}}-master" state=restarted
+    when: not openshift_master_ha | bool
+
+  - name: Restart master cluster
+    command: pcs resource restart master
+    when: openshift_master_ha | bool
+    run_once: true
+
+  - name: Wait for the clustered master service to be available
+    wait_for:
+      host: "{{ openshift_master_cluster_vip }}"
+      port: 8443
+      state: started
+      timeout: 180
+      delay: 90
+    when: openshift_master_ha | bool
+    run_once: true
+
+
+###############################################################################
+# Post upgrade - Upgrade default router, default registry and examples
+###############################################################################
+- name: Upgrade default router and default registry
+  hosts: oo_first_master
   vars:
     openshift_examples_import_command: "update"
     openshift_deployment_type: "{{ deployment_type }}"
+    registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + g_new_version  ) }}"
+    router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
+    oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
   roles:
-    - openshift_examples
+  - openshift_examples
+  pre_tasks:
+  - name: Check for default router
+    command: >
+      {{ oc_cmd }} get -n default dc/router
+    register: _default_router
+    failed_when: false
+    changed_when: false
 
-- name: Ensure master services enabled
-  hosts: masters
-  vars:
-    openshift_master_ha: "{{ groups['masters'] | length > 1 }}"
-  tasks:
-  - name: Enable master services
-    service: name="{{ openshift.common.service_type}}-master" state=started enabled=yes
-    when: not openshift_master_ha | bool
+  - name: Check for allowHostNetwork and allowHostPorts
+    when: _default_router.rc == 0
+    shell: >
+      {{ oc_cmd }} get -o yaml scc/privileged | /usr/bin/grep -e allowHostPorts -e allowHostNetwork
+    register: _scc
 
-- name: Ensure node services enabled
-  hosts: nodes
-  tasks:
-    - name: Restart node services
-      service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
+  - name: Grant allowHostNetwork and allowHostPorts
+    when:
+    - _default_router.rc == 0
+    - "'false' in _scc.stdout"
+    command: >
+      {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
 
+  - name: Update deployment config to 1.0.4/3.0.1 spec
+    when: _default_router.rc == 0
+    command: >
+      {{ oc_cmd }} patch dc/router -p
+      '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
+
+  - name: Switch to hostNetwork=true
+    when: _default_router.rc == 0
+    command: >
+      {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
+
+  - name: Update router image to current version
+    when: _default_router.rc == 0
+    command: >
+      {{ oc_cmd }} patch dc/router -p
+      '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
+
+  - name: Check for default registry
+    command: >
+      {{ oc_cmd }} get -n default dc/docker-registry
+    register: _default_registry
+    failed_when: false
+    changed_when: false
+
+  - name: Update registry image to current version
+    when: _default_registry.rc == 0
+    command: >
+      {{ oc_cmd }} patch dc/docker-registry -p
+      '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
-- 
cgit v1.2.3


From 35e52819020fbf6925bbc7b43c9a5ca40e3eb6a5 Mon Sep 17 00:00:00 2001
From: Jason DeTiberus <jdetiber@redhat.com>
Date: Wed, 11 Nov 2015 15:27:57 -0500
Subject: Fix update error for templates that didn't previously exist

---
 .../common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml   | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'playbooks/common')

diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index 1cd7327cb..78797f8b8 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -363,13 +363,20 @@
 - name: Upgrade default router and default registry
   hosts: oo_first_master
   vars:
-    openshift_examples_import_command: "update"
     openshift_deployment_type: "{{ deployment_type }}"
     registry_image: "{{  openshift.master.registry_url | replace( '${component}', 'docker-registry' )  | replace ( '${version}', 'v' + g_new_version  ) }}"
     router_image: "{{ openshift.master.registry_url | replace( '${component}', 'haproxy-router' ) | replace ( '${version}', 'v' + g_new_version ) }}"
     oc_cmd: "{{ openshift.common.client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig"
   roles:
+  # Create the new templates shipped in 3.1, existing templates are left
+  # unmodified. This prevents the subsequent role definition for
+  # openshift_examples from failing when trying to replace templates that do
+  # not already exist. We could have potentially done a replace --force to
+  # create and update in one step.
   - openshift_examples
+  # Update the existing templates
+  - role: openshift_examples
+    openshift_examples_import_command: replace
   pre_tasks:
   - name: Check for default router
     command: >
-- 
cgit v1.2.3