Merge remote-tracking branch 'upstream/master' into upgrade33

33 files changed, 495 insertions, 260 deletions

diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml
deleted file mode 100644
index 837fa67db..000000000
--- a/roles/haproxy/tasks/main.yml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-- name: Install haproxy
-  action: "{{ ansible_pkg_mgr }} name=haproxy state=present"
-  when: not openshift.common.is_containerized | bool
-
-- name: Configure systemd service directory for haproxy
-  file:
-    path: /etc/systemd/system/haproxy.service.d
-    state: directory
-  when: haproxy_limit_nofile is defined
-
-- name: Configure the nofile limits for haproxy
-  ini_file:
-    dest: /etc/systemd/system/haproxy.service.d/limits.conf
-    section: Service
-    option: LimitNOFILE
-    value: "{{ haproxy_limit_nofile }}"
-  when: haproxy_limit_nofile is defined
-  notify: restart haproxy
-  register: nofile_limit_result
-
-- name: Reload systemd if needed
-  command: systemctl daemon-reload
-  when: nofile_limit_result | changed
-
-- name: Configure haproxy
-  template:
-    src: haproxy.cfg.j2
-    dest: /etc/haproxy/haproxy.cfg
-    owner: root
-    group: root
-    mode: 0644
-  notify: restart haproxy
-
-- name: Enable and start haproxy
-  service:
-    name: haproxy
-    state: started
-    enabled: yes
-  register: start_result
-
-- set_fact:
-    haproxy_start_result_changed: "{{ start_result | changed }}"
diff --git a/roles/openshift_ansible_inventory/README.md b/roles/openshift_ansible_inventory/README.md
deleted file mode 100644
index b62287c12..000000000
--- a/roles/openshift_ansible_inventory/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-OpenShift Ansible Inventory
-=========
-
-Install and configure openshift-ansible-inventory.
-
-Requirements
-------------
-
-None
-
-Role Variables
---------------
-
-oo_inventory_group
-oo_inventory_user
-oo_inventory_accounts
-oo_inventory_cache_max_age
-
-Dependencies
-------------
-
-None
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
-    - hosts: servers
-      roles:
-         - { role: username.rolename, x: 42 }
-
-License
--------
-
-ASL 2.0
-
-Author Information
-------------------
-
-OpenShift operations, Red Hat, Inc
diff --git a/roles/openshift_ansible_inventory/defaults/main.yml b/roles/openshift_ansible_inventory/defaults/main.yml
deleted file mode 100644
index f53c00c80..000000000
--- a/roles/openshift_ansible_inventory/defaults/main.yml
+++ /dev/null
@@ -1,4 +0,0 @@
----
-oo_inventory_group: root
-oo_inventory_owner: root
-oo_inventory_cache_max_age: 1800
diff --git a/roles/openshift_ansible_inventory/handlers/main.yml b/roles/openshift_ansible_inventory/handlers/main.yml
deleted file mode 100644
index e2db43477..000000000
--- a/roles/openshift_ansible_inventory/handlers/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-# handlers file for openshift_ansible_inventory
diff --git a/roles/openshift_ansible_inventory/meta/main.yml b/roles/openshift_ansible_inventory/meta/main.yml
deleted file mode 100644
index 7f7387e80..000000000
--- a/roles/openshift_ansible_inventory/meta/main.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-galaxy_info:
-  author: OpenShift
-  description:  Install and configure openshift-ansible-inventory
-  company: Red Hat, Inc
-  license: ASL 2.0
-  min_ansible_version: 1.2
-dependencies: []
diff --git a/roles/openshift_ansible_inventory/tasks/main.yml b/roles/openshift_ansible_inventory/tasks/main.yml
deleted file mode 100644
index 05c7a5f93..000000000
--- a/roles/openshift_ansible_inventory/tasks/main.yml
+++ /dev/null
@@ -1,47 +0,0 @@
----
-- action: "{{ ansible_pkg_mgr }} name={{ item}} state=present"
-  with_items:
-  - openshift-ansible-inventory
-  - openshift-ansible-inventory-aws
-  - openshift-ansible-inventory-gce
-  when: not openshift.common.is_containerized | bool
-
-- name:
-  copy:
-    content: "{{ oo_inventory_accounts | to_nice_yaml }}"
-    dest: /etc/ansible/multi_inventory.yaml
-    group: "{{ oo_inventory_group }}"
-    owner: "{{ oo_inventory_owner }}"
-    mode: "0640"
-
-- file:
-    state: directory
-    dest: /etc/ansible/inventory
-    owner: root
-    group: libra_ops
-    mode: 0750
-
-- file:
-    state: link
-    src: /usr/share/ansible/inventory/multi_inventory.py
-    dest: /etc/ansible/inventory/multi_inventory.py
-    owner: root
-    group: libra_ops
-
-# This cron uses the above location to call its job
-- name: Cron to keep cache fresh
-  cron:
-    name: 'multi_inventory'
-    minute: '*/10'
-    job: '/usr/share/ansible/inventory/multi_inventory.py --refresh-cache &> /dev/null'
-  when: oo_cron_refresh_cache is defined and oo_cron_refresh_cache
-
-- name: Set cache location
-  file:
-    state: directory
-    dest: "{{ oo_inventory_cache_location | dirname }}"
-    owner: root
-    group: libra_ops
-    recurse: yes
-    mode: '2770'
-  when: oo_inventory_cache_location is defined
diff --git a/roles/openshift_ansible_inventory/vars/main.yml b/roles/openshift_ansible_inventory/vars/main.yml
deleted file mode 100644
index 25c049282..000000000
--- a/roles/openshift_ansible_inventory/vars/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
----
-# vars file for openshift_ansible_inventory
diff --git a/roles/openshift_ca/README.md b/roles/openshift_ca/README.md
new file mode 100644
index 000000000..96c9cd5f2
--- /dev/null
+++ b/roles/openshift_ca/README.md
@@ -0,0 +1,48 @@
+OpenShift CA
+============
+
+This role delegates all tasks to the `openshift_ca_host` such that this role can be depended on by other OpenShift certificate roles.
+
+Requirements
+------------
+
+Role Variables
+--------------
+
+From this role:
+
+| Name                    | Default value                                 | Description                                                                 |
+|-------------------------|-----------------------------------------------|-----------------------------------------------------------------------------|
+| openshift_ca_host       | None (Required)                               | The hostname of the system where the OpenShift CA will be created.          |
+| openshift_ca_config_dir | `{{ openshift.common.config_base }}/master`   | CA certificate directory.                                                   |
+| openshift_ca_cert       | `{{ openshift_ca_config_dir }}/ca.crt`        | CA certificate path including CA certificate filename.                      |
+| openshift_ca_key        | `{{ openshift_ca_config_dir }}/ca.key`        | CA key path including CA key filename.                                      |
+| openshift_ca_serial     | `{{ openshift_ca_config_dir }}/ca.serial.txt` | CA serial path including CA serial filename.                                |
+| openshift_version       | `{{ openshift_pkg_version }}`                 | OpenShift package version.                                                  |
+
+Dependencies
+------------
+
+* openshift_repos
+* openshift_cli
+
+Example Playbook
+----------------
+
+```
+- name: Create OpenShift CA
+  hosts: localhost
+  roles:
+  - role: openshift_ca
+    openshift_ca_host: master1.example.com
+```
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/openshift_master_ca/meta/main.yml b/roles/openshift_ca/meta/main.yml
index b5dd466c9..0089f4209 100644
--- a/roles/openshift_master_ca/meta/main.yml
+++ b/roles/openshift_ca/meta/main.yml
@@ -1,10 +1,10 @@
 ---
 galaxy_info:
   author: Jason DeTiberus
-  description:
+  description: OpenShift CA
   company: Red Hat, Inc.
   license: Apache License, Version 2.0
-  min_ansible_version: 1.8
+  min_ansible_version: 1.9.4
   platforms:
   - name: EL
     versions:
@@ -13,5 +13,5 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- { role: openshift_repos }
-- { role: openshift_cli }
+- role: openshift_repos
+- role: openshift_cli
diff --git a/roles/openshift_ca/tasks/main.yml b/roles/openshift_ca/tasks/main.yml
new file mode 100644
index 000000000..497473f22
--- /dev/null
+++ b/roles/openshift_ca/tasks/main.yml
@@ -0,0 +1,56 @@
+---
+- fail:
+    msg: "openshift_ca_host variable must be defined for this role"
+  when: openshift_ca_host is not defined
+
+- name: Install the base package for admin tooling
+  action: >
+    {{ ansible_pkg_mgr }}
+    name={{ openshift.common.service_type }}{{ openshift_version  }}
+    state=present
+  when: not openshift.common.is_containerized | bool
+  register: install_result
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- name: Reload generated facts
+  openshift_facts:
+  when: install_result | changed
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- name: Create openshift_ca_config_dir if it does not exist
+  file:
+    path: "{{ openshift_ca_config_dir }}"
+    state: directory
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- name: Determine if CA must be created
+  stat:
+    path: "{{ openshift_ca_config_dir }}/{{ item }}"
+  register: g_master_ca_stat_result
+  with_items:
+  - ca.crt
+  - ca.key
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- set_fact:
+    master_ca_missing: "{{ False in (g_master_ca_stat_result.results
+                           | oo_collect(attribute='stat.exists')
+                           | list) }}"
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
+
+- name: Create the master certificates if they do not already exist
+  command: >
+    {{ openshift.common.admin_binary }} create-master-certs
+      --hostnames={{ openshift_master_hostnames | join(',') }}
+      --master={{ openshift.master.api_url }}
+      --public-master={{ openshift.master.public_api_url }}
+      --cert-dir={{ openshift_ca_config_dir }}
+      --overwrite=false
+  when: hostvars[openshift_ca_host].master_ca_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
+  run_once: true
diff --git a/roles/openshift_ca/vars/main.yml b/roles/openshift_ca/vars/main.yml
new file mode 100644
index 000000000..a32e385ec
--- /dev/null
+++ b/roles/openshift_ca/vars/main.yml
@@ -0,0 +1,6 @@
+---
+openshift_ca_config_dir: "{{ openshift.common.config_base }}/master"
+openshift_ca_cert: "{{ openshift_ca_config_dir }}/ca.crt"
+openshift_ca_key: "{{ openshift_ca_config_dir }}/ca.key"
+openshift_ca_serial: "{{ openshift_ca_config_dir }}/ca.serial.txt"
+openshift_version: "{{ openshift_pkg_version | default('') }}"
diff --git a/roles/openshift_clock/meta/main.yml b/roles/openshift_clock/meta/main.yml
new file mode 100644
index 000000000..3e175beb0
--- /dev/null
+++ b/roles/openshift_clock/meta/main.yml
@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: Jeremiah Stuever
+  description: OpenShift Clock
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+dependencies:
+- { role: openshift_facts }
diff --git a/roles/openshift_clock/tasks/main.yaml b/roles/openshift_clock/tasks/main.yaml
new file mode 100644
index 000000000..5a8403f68
--- /dev/null
+++ b/roles/openshift_clock/tasks/main.yaml
@@ -0,0 +1,14 @@
+---
+- name: Set clock facts
+  openshift_facts:
+    role: clock
+    local_facts:
+      enabled: "{{ openshift_clock_enabled | default(None) }}"
+
+- name: Install ntp package
+  action: "{{ ansible_pkg_mgr }} name=ntp state=present"
+  when: openshift.clock.enabled | bool and not openshift.clock.chrony_installed | bool
+
+- name: Start and enable ntpd/chronyd
+  shell: timedatectl set-ntp true
+  when: openshift.clock.enabled | bool
diff --git a/roles/openshift_etcd/meta/main.yml b/roles/openshift_etcd/meta/main.yml
index 7cc548f69..de36b201b 100644
--- a/roles/openshift_etcd/meta/main.yml
+++ b/roles/openshift_etcd/meta/main.yml
@@ -13,6 +13,7 @@ galaxy_info:
   - cloud
 dependencies:
 - role: openshift_etcd_facts
+- role: openshift_clock
 - role: openshift_docker
   when: openshift.common.is_containerized | bool
 - role: etcd
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index eee4d2b8e..4e7785cd7 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -1549,11 +1549,13 @@ class OpenShiftFacts(object):
             OpenShiftFactsUnsupportedRoleError:
     """
     known_roles = ['builddefaults',
+                   'clock',
                    'cloudprovider',
                    'common',
                    'docker',
                    'etcd',
                    'hosted',
+                   'loadbalancer',
                    'master',
                    'node']
 
@@ -1719,6 +1721,16 @@ class OpenShiftFacts(object):
                 docker['version'] = version_info['version']
             defaults['docker'] = docker
 
+        if 'clock' in roles:
+            exit_code, _, _ = module.run_command(['rpm', '-q', 'chrony'])
+            if exit_code == 0:
+                chrony_installed = True
+            else:
+                chrony_installed = False
+            defaults['clock'] = dict(
+                enabled=True,
+                chrony_installed=chrony_installed)
+
         if 'cloudprovider' in roles:
             defaults['cloudprovider'] = dict(kind=None)
 
@@ -1763,6 +1775,13 @@ class OpenShiftFacts(object):
                 router=dict()
             )
 
+        if 'loadbalancer' in roles:
+            loadbalancer = dict(frontend_port='8443',
+                                default_maxconn='20000',
+                                global_maxconn='20000',
+                                limit_nofile='100000')
+            defaults['loadbalancer'] = loadbalancer
+
         return defaults
 
     def guess_host_provider(self):
diff --git a/roles/haproxy/README.md b/roles/openshift_loadbalancer/README.md
index 5bc415066..81fc282be 100644
--- a/roles/haproxy/README.md
+++ b/roles/openshift_loadbalancer/README.md
@@ -1,5 +1,5 @@
-HAProxy
-=======
+OpenShift HAProxy Loadbalancer
+==============================
 
 TODO
 
diff --git a/roles/haproxy/defaults/main.yml b/roles/openshift_loadbalancer/defaults/main.yml
index a1524cfe1..a1524cfe1 100644
--- a/roles/haproxy/defaults/main.yml
+++ b/roles/openshift_loadbalancer/defaults/main.yml
diff --git a/roles/haproxy/handlers/main.yml b/roles/openshift_loadbalancer/handlers/main.yml
index 5b8691b26..5b8691b26 100644
--- a/roles/haproxy/handlers/main.yml
+++ b/roles/openshift_loadbalancer/handlers/main.yml
diff --git a/roles/haproxy/meta/main.yml b/roles/openshift_loadbalancer/meta/main.yml
index 0fad106a9..fe336acf7 100644
--- a/roles/haproxy/meta/main.yml
+++ b/roles/openshift_loadbalancer/meta/main.yml
@@ -1,7 +1,7 @@
 ---
 galaxy_info:
   author: Jason DeTiberus
-  description: HAProxy
+  description: OpenShift haproxy loadbalancer
   company: Red Hat, Inc.
   license: Apache License, Version 2.0
   min_ansible_version: 1.9
@@ -10,5 +10,6 @@ galaxy_info:
     versions:
     - 7
 dependencies:
-- { role: os_firewall }
-- { role: openshift_repos }
+- role: openshift_facts
+- role: os_firewall
+- role: openshift_repos
diff --git a/roles/openshift_loadbalancer/tasks/main.yml b/roles/openshift_loadbalancer/tasks/main.yml
new file mode 100644
index 000000000..5514aa70b
--- /dev/null
+++ b/roles/openshift_loadbalancer/tasks/main.yml
@@ -0,0 +1,73 @@
+---
+- name: Set haproxy frontend port
+  openshift_facts:
+    role: loadbalancer
+    local_facts:
+      frontend_port: "{{ openshift_master_api_port | default(None) }}"
+
+- name: Set loadbalancer facts
+  openshift_facts:
+    role: loadbalancer
+    local_facts:
+      limit_nofile: "{{ openshift_loadbalancer_limit_nofile | default(None) }}"
+      default_maxconn: "{{ openshift_loadbalancer_default_maxconn | default(None) }}"
+      global_maxconn: "{{ openshift_loadbalancer_global_maxconn | default(None) }}"
+      frontends:
+      - name: atomic-openshift-api
+        mode: tcp
+        options:
+        - tcplog
+        binds:
+        - "*:{{ openshift.loadbalancer.frontend_port }}"
+        default_backend: atomic-openshift-api
+      backends:
+      - name: atomic-openshift-api
+        mode: tcp
+        option: tcplog
+        balance: source
+        servers: "{{ hostvars
+                     | oo_select_keys(groups['oo_masters'])
+                     | oo_haproxy_backend_masters(openshift.loadbalancer.frontend_port) }}"
+
+- name: Install haproxy
+  action: "{{ ansible_pkg_mgr }} name=haproxy state=present"
+  when: not openshift.common.is_containerized | bool
+
+- name: Configure systemd service directory for haproxy
+  file:
+    path: /etc/systemd/system/haproxy.service.d
+    state: directory
+  when: "'limit_nofile' in openshift.loadbalancer"
+
+- name: Configure the nofile limits for haproxy
+  ini_file:
+    dest: /etc/systemd/system/haproxy.service.d/limits.conf
+    section: Service
+    option: LimitNOFILE
+    value: "{{ openshift.loadbalancer.limit_nofile }}"
+  when: "'limit_nofile' in openshift.loadbalancer"
+  notify: restart haproxy
+  register: nofile_limit_result
+
+- name: Reload systemd if needed
+  command: systemctl daemon-reload
+  when: nofile_limit_result | changed
+
+- name: Configure haproxy
+  template:
+    src: haproxy.cfg.j2
+    dest: /etc/haproxy/haproxy.cfg
+    owner: root
+    group: root
+    mode: 0644
+  notify: restart haproxy
+
+- name: Enable and start haproxy
+  service:
+    name: haproxy
+    state: started
+    enabled: yes
+  register: start_result
+
+- set_fact:
+    haproxy_start_result_changed: "{{ start_result | changed }}"
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2
index cb4380971..05e360d3b 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2
@@ -3,7 +3,7 @@
 global
     chroot      /var/lib/haproxy
     pidfile     /var/run/haproxy.pid
-    maxconn     {{ haproxy_global_maxconn | default('4000') }}
+    maxconn     {{ openshift.loadbalancer.global_maxconn }}
     user        haproxy
     group       haproxy
     daemon
@@ -31,14 +31,14 @@ defaults
     timeout server          300s
     timeout http-keep-alive 10s
     timeout check           10s
-    maxconn                 {{ haproxy_default_maxconn | default('3000') }}
+    maxconn                 {{ openshift.loadbalancer.default_maxconn }}
 
 listen stats :9000
     mode http
     stats enable
     stats uri /
 
-{% for frontend in haproxy_frontends %}
+{% for frontend in openshift.loadbalancer.frontends %}
 frontend  {{ frontend.name }}
 {% for bind in frontend.binds %}
     bind {{ bind }}
@@ -59,7 +59,7 @@ frontend  {{ frontend.name }}
 {% endif %}
 {% endfor %}
 
-{% for backend in haproxy_backends %}
+{% for backend in openshift.loadbalancer.backends %}
 backend {{ backend.name }}
     balance {{ backend.balance }}
 {% if 'mode' in backend %}
diff --git a/roles/openshift_master/meta/main.yml b/roles/openshift_master/meta/main.yml
index d8834d27f..f6b926d74 100644
--- a/roles/openshift_master/meta/main.yml
+++ b/roles/openshift_master/meta/main.yml
@@ -12,8 +12,10 @@ galaxy_info:
   categories:
   - cloud
 dependencies:
+- role: openshift_clock
 - role: openshift_docker
 - role: openshift_cli
+- role: openshift_master_certificates
 - role: openshift_cloud_provider
 - role: openshift_builddefaults
 - role: openshift_master_facts
diff --git a/roles/openshift_master_ca/README.md b/roles/openshift_master_ca/README.md
deleted file mode 100644
index 5b2d3601b..000000000
--- a/roles/openshift_master_ca/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-OpenShift Master CA
-========================
-
-TODO
-
-Requirements
-------------
-
-TODO
-
-Role Variables
---------------
-
-TODO
-
-Dependencies
-------------
-
-TODO
-
-Example Playbook
-----------------
-
-TODO
-
-License
--------
-
-Apache License Version 2.0
-
-Author Information
-------------------
-
-Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/openshift_master_certificates/README.md b/roles/openshift_master_certificates/README.md
index ba3d5f28c..a80d47040 100644
--- a/roles/openshift_master_certificates/README.md
+++ b/roles/openshift_master_certificates/README.md
@@ -1,27 +1,44 @@
 OpenShift Master Certificates
 ========================
 
-TODO
+This role determines if OpenShift master certificates must be created, delegates certificate creation to the `openshift_ca_host` and then deploys those certificates to master hosts which this role is being applied to. If this role is applied to the `openshift_ca_host`, certificate deployment will be skipped.
 
 Requirements
 ------------
 
-TODO
-
 Role Variables
 --------------
 
-TODO
+From `openshift_ca`:
+
+| Name                                  | Default value                                                             | Description                                                                                                                   |
+|---------------------------------------|---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| openshift_ca_host                     | None (Required)                                                           | The hostname of the system where the OpenShift CA will be (or has been) created.                                              |
+
+From this role:
+
+| Name                                  | Default value                                                             | Description                                                                                                                   |
+|---------------------------------------|---------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| openshift_generated_configs_dir       | `{{ openshift.common.config_base }}/generated-configs`                    | Directory in which per-master generated config directories will be created on the `openshift_ca_host`.                        |
+| openshift_master_cert_subdir          | `master-{{ openshift.common.hostname }}`                                  | Directory within `openshift_generated_configs_dir` where per-master configurations will be placed on the `openshift_ca_host`. |
+| openshift_master_config_dir           | `{{ openshift.common.config_base }}/master`                               | Master configuration directory in which certificates will be deployed on masters.                                             |
+| openshift_master_generated_config_dir | `{{ openshift_generated_configs_dir }}/{{ openshift_master_cert_subdir }` | Full path to the per-master generated config directory.                                                                       |
 
 Dependencies
 ------------
 
-TODO
+* openshift_ca
 
 Example Playbook
 ----------------
 
-TODO
+```
+- name: Create OpenShift Master Certificates
+  hosts: masters
+  roles:
+  - role: openshift_master_certificates
+    openshift_ca_host: master1.example.com
+```
 
 License
 -------
diff --git a/roles/openshift_master_certificates/meta/main.yml b/roles/openshift_master_certificates/meta/main.yml
index fd7b73b0f..90fc0fb10 100644
--- a/roles/openshift_master_certificates/meta/main.yml
+++ b/roles/openshift_master_certificates/meta/main.yml
@@ -1,10 +1,10 @@
 ---
 galaxy_info:
   author: Jason DeTiberus
-  description:
+  description: OpenShift Master Certificates
   company: Red Hat, Inc.
   license: Apache License, Version 2.0
-  min_ansible_version: 1.8
+  min_ansible_version: 1.9.4
   platforms:
   - name: EL
     versions:
@@ -13,4 +13,4 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- { role: openshift_master_ca }
+- role: openshift_ca
diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml
index 394f9d381..dd105652b 100644
--- a/roles/openshift_master_certificates/tasks/main.yml
+++ b/roles/openshift_master_certificates/tasks/main.yml
@@ -1,38 +1,121 @@
 ---
+- set_fact:
+    openshift_master_certs_no_etcd:
+    - admin.crt
+    - master.kubelet-client.crt
+    - "{{ 'master.proxy-client.crt' if openshift.common.version_gte_3_1_or_1_1 else omit }}"
+    - master.server.crt
+    - openshift-master.crt
+    - openshift-registry.crt
+    - openshift-router.crt
+    - etcd.server.crt
+    openshift_master_certs_etcd:
+    - master.etcd-client.crt
+
+- set_fact:
+    openshift_master_certs: "{{ (openshift_master_certs_no_etcd | union(openshift_master_certs_etcd )) if openshift_master_etcd_hosts | length > 0 else openshift_master_certs_no_etcd }}"
+
+- name: Check status of master certificates
+  stat:
+    path: "{{ openshift_master_config_dir }}/{{ item }}"
+  with_items:
+  - "{{ openshift_master_certs }}"
+  register: g_master_cert_stat_result
+
+- set_fact:
+    master_certs_missing: "{{ False in (g_master_cert_stat_result.results
+                              | oo_collect(attribute='stat.exists')
+                              | list) }}"
+
 - name: Ensure the generated_configs directory present
   file:
-    path: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}"
+    path: "{{ openshift_master_generated_config_dir }}"
     state: directory
     mode: 0700
-  with_items: "{{ masters_needing_certs | default([]) }}"
+  when: master_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
 
 - file:
-    src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
-    dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+    src: "{{ openshift_master_config_dir }}/{{ item }}"
+    dest: "{{ openshift_master_generated_config_dir }}/{{ item }}"
     state: hard
-  with_nested:
-  - "{{ masters_needing_certs | default([]) }}"
-  -
-    - ca.crt
-    - ca.key
-    - ca.serial.txt
+  with_items:
+  - ca.crt
+  - ca.key
+  - ca.serial.txt
+  when: master_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
 
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ item.openshift.common.all_hostnames | join(',') }}
-      --master={{ item.openshift.master.api_url }}
-      --public-master={{ item.openshift.master.public_api_url }}
-      --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
+      --hostnames={{ openshift.common.all_hostnames | join(',') }}
+      --master={{ openshift.master.api_url }}
+      --public-master={{ openshift.master.public_api_url }}
+      --cert-dir={{ openshift_master_generated_config_dir }}
       --overwrite=false
-  when: item.master_certs_missing | bool
-  with_items: "{{ masters_needing_certs | default([]) }}"
+  when: master_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
 
 - file:
-    src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
-    dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+    src: "{{ openshift_master_config_dir }}/{{ item }}"
+    dest: "{{ openshift_master_generated_config_dir }}/{{ item }}"
     state: hard
     force: true
-  with_nested:
-  - "{{ masters_needing_certs | default([]) }}"
+  with_items:
   - "{{ hostvars[inventory_hostname] | certificates_to_synchronize }}"
+  when: master_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Remove generated etcd client certs when using external etcd
+  file:
+    path: "{{ openshift_master_generated_config_dir }}/{{ item }}"
+    state: absent
+  when: openshift_master_etcd_hosts | length > 0
+  with_items:
+  - master.etcd-client.crt
+  - master.etcd-client.key
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Create local temp directory for syncing certs
+  local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+  register: g_master_mktemp
+  changed_when: False
+  when: master_certs_missing | bool
+  delegate_to: localhost
+
+- name: Create a tarball of the master certs
+  command: >
+    tar -czvf {{ openshift_master_generated_config_dir }}.tgz
+      -C {{ openshift_master_generated_config_dir }} .
+  args:
+    creates: "{{ openshift_master_generated_config_dir }}.tgz"
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Retrieve the master cert tarball from the master
+  fetch:
+    src: "{{ openshift_master_generated_config_dir }}.tgz"
+    dest: "{{ g_master_mktemp.stdout }}/"
+    flat: yes
+    fail_on_missing: yes
+    validate_checksum: yes
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Ensure certificate directory exists
+  file:
+    path: "{{ openshift_master_config_dir }}"
+    state: directory
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
+
+- name: Unarchive the tarball on the master
+  unarchive:
+    src: "{{ g_master_mktemp.stdout }}/{{ openshift_master_cert_subdir }}.tgz"
+    dest: "{{ openshift_master_config_dir }}"
+  when: master_certs_missing | bool and inventory_hostname != openshift_ca_host
+
+- file: name={{ g_master_mktemp.stdout }} state=absent
+  changed_when: False
+  when: master_certs_missing | bool
+  delegate_to: localhost
diff --git a/roles/openshift_master_certificates/vars/main.yml b/roles/openshift_master_certificates/vars/main.yml
index 3f18ddc79..66f2e5162 100644
--- a/roles/openshift_master_certificates/vars/main.yml
+++ b/roles/openshift_master_certificates/vars/main.yml
@@ -1,3 +1,5 @@
 ---
 openshift_generated_configs_dir: "{{ openshift.common.config_base }}/generated-configs"
+openshift_master_cert_subdir: "master-{{ openshift.common.hostname }}"
 openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
+openshift_master_generated_config_dir: "{{ openshift_generated_configs_dir }}/{{ openshift_master_cert_subdir }}"
diff --git a/roles/openshift_node/meta/main.yml b/roles/openshift_node/meta/main.yml
index db1776632..ea52bbb99 100644
--- a/roles/openshift_node/meta/main.yml
+++ b/roles/openshift_node/meta/main.yml
@@ -12,10 +12,11 @@ galaxy_info:
   categories:
   - cloud
 dependencies:
+- role: openshift_clock
 - role: openshift_docker
+- role: openshift_node_certificates
 - role: openshift_cloud_provider
 - role: openshift_common
 - role: openshift_node_dnsmasq
   when: openshift.common.use_dnsmasq
 - role: os_firewall
-
diff --git a/roles/openshift_node_certificates/README.md b/roles/openshift_node_certificates/README.md
index 6264d253a..f56066b29 100644
--- a/roles/openshift_node_certificates/README.md
+++ b/roles/openshift_node_certificates/README.md
@@ -1,27 +1,44 @@
-OpenShift/Atomic Enterprise Node Certificates
-=============================================
+OpenShift Node Certificates
+===========================
 
-TODO
+This role determines if OpenShift node certificates must be created, delegates certificate creation to the `openshift_ca_host` and then deploys those certificates to node hosts which this role is being applied to.
 
 Requirements
 ------------
 
-TODO
-
 Role Variables
 --------------
 
-TODO
+From `openshift_ca`:
+
+| Name                                | Default value                                                           | Description                                                                                                               |
+|-------------------------------------|-------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
+| openshift_ca_host                   | None (Required)                                                         | The hostname of the system where the OpenShift CA will be (or has been) created.                                          |
+
+From this role:
+
+| Name                                | Default value                                                           | Description                                                                                                               |
+|-------------------------------------|-------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
+| openshift_generated_configs_dir     | `{{ openshift.common.config_base }}/generated-configs`                  | Directory in which per-node generated config directories will be created on the `openshift_ca_host`.                      |
+| openshift_node_cert_subdir          | `node-{{ openshift.common.hostname }}`                                  | Directory within `openshift_generated_configs_dir` where per-node certificates will be placed on the `openshift_ca_host`. |
+| openshift_node_config_dir           | `{{ openshift.common.config_base }}/node`                               | Node configuration directory in which certificates will be deployed on nodes.                                             |
+| openshift_node_generated_config_dir | `{{ openshift_generated_configs_dir }}/{{ openshift_node_cert_subdir }` | Full path to the per-node generated config directory.                                                                     |
 
 Dependencies
 ------------
 
-TODO
+* openshift_ca
 
 Example Playbook
 ----------------
 
-TODO
+```
+- name: Create OpenShift Node Certificates
+  hosts: nodes
+  roles:
+  - role: openshift_node_certificates
+    openshift_ca_host: master1.example.com
+```
 
 License
 -------
diff --git a/roles/openshift_node_certificates/meta/main.yml b/roles/openshift_node_certificates/meta/main.yml
index f3236e850..3caa1cdf1 100644
--- a/roles/openshift_node_certificates/meta/main.yml
+++ b/roles/openshift_node_certificates/meta/main.yml
@@ -1,10 +1,10 @@
 ---
 galaxy_info:
   author: Jason DeTiberus
-  description:
+  description: OpenShift Node Certificates
   company: Red Hat, Inc.
   license: Apache License, Version 2.0
-  min_ansible_version: 1.8
+  min_ansible_version: 1.9.4
   platforms:
   - name: EL
     versions:
@@ -13,4 +13,4 @@ galaxy_info:
   - cloud
   - system
 dependencies:
-- { role: openshift_facts }
+- role: openshift_ca
diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml
index 216c11093..147a432a4 100644
--- a/roles/openshift_node_certificates/tasks/main.yml
+++ b/roles/openshift_node_certificates/tasks/main.yml
@@ -1,36 +1,95 @@
 ---
-- name: Create openshift_generated_configs_dir if it doesn\'t exist
+- name: Check status of node certificates
+  stat:
+    path: "{{ openshift.common.config_base }}/node/{{ item }}"
+  with_items:
+  - "system:node:{{ openshift.common.hostname }}.crt"
+  - "system:node:{{ openshift.common.hostname }}.key"
+  - "system:node:{{ openshift.common.hostname }}.kubeconfig"
+  - ca.crt
+  - server.key
+  - server.crt
+  register: g_node_cert_stat_result
+
+- set_fact:
+    node_certs_missing: "{{ False in (g_node_cert_stat_result.results
+                            | oo_collect(attribute='stat.exists')
+                            | list) }}"
+
+- name: Create openshift_generated_configs_dir if it does not exist
   file:
     path: "{{ openshift_generated_configs_dir }}"
     state: directory
     mode: 0700
-  when: nodes_needing_certs | length > 0
+  when: node_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
 
 - name: Generate the node client config
   command: >
     {{ openshift.common.admin_binary }} create-api-client-config
-      --certificate-authority={{ openshift_master_ca_cert }}
-      --client-dir={{ openshift_generated_configs_dir }}/node-{{ item.openshift.common.hostname }}
+      --certificate-authority={{ openshift_ca_cert }}
+      --client-dir={{ openshift_node_generated_config_dir }}
       --groups=system:nodes
-      --master={{ openshift.master.api_url }}
-      --signer-cert={{ openshift_master_ca_cert }}
-      --signer-key={{ openshift_master_ca_key }}
-      --signer-serial={{ openshift_master_ca_serial }}
-      --user=system:node:{{ item.openshift.common.hostname }}
+      --master={{ hostvars[openshift_ca_host].openshift.master.api_url }}
+      --signer-cert={{ openshift_ca_cert }}
+      --signer-key={{ openshift_ca_key }}
+      --signer-serial={{ openshift_ca_serial }}
+      --user=system:node:{{ openshift.common.hostname }}
   args:
-    creates: "{{ openshift_generated_configs_dir }}/node-{{ item.openshift.common.hostname }}"
-  with_items: "{{ nodes_needing_certs | default([]) }}"
+    creates: "{{ openshift_node_generated_config_dir }}"
+  when: node_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
 
 - name: Generate the node server certificate
   command: >
     {{ openshift.common.admin_binary }} ca create-server-cert
-      --cert={{ openshift_generated_configs_dir }}/node-{{ item.openshift.common.hostname }}/server.crt
-      --key={{ openshift_generated_configs_dir }}/node-{{ item.openshift.common.hostname }}/server.key
+      --cert={{ openshift_node_generated_config_dir }}/server.crt
+      --key={{ openshift_generated_configs_dir }}/node-{{ openshift.common.hostname }}/server.key
       --overwrite=true
-      --hostnames={{ item.openshift.common.all_hostnames |join(",") }}
-      --signer-cert={{ openshift_master_ca_cert }}
-      --signer-key={{ openshift_master_ca_key }}
-      --signer-serial={{ openshift_master_ca_serial }}
+      --hostnames={{ openshift.common.all_hostnames |join(",") }}
+      --signer-cert={{ openshift_ca_cert }}
+      --signer-key={{ openshift_ca_key }}
+      --signer-serial={{ openshift_ca_serial }}
+  args:
+    creates: "{{ openshift_node_generated_config_dir }}/server.crt"
+  when: node_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host}}"
+
+- name: Create local temp directory for syncing certs
+  local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+  register: node_cert_mktemp
+  changed_when: False
+  when: node_certs_missing | bool
+  delegate_to: localhost
+
+- name: Create a tarball of the node config directories
+  command: >
+    tar -czvf {{ openshift_node_generated_config_dir }}.tgz
+    --transform 's|system:{{ openshift_node_cert_subdir }}|node|'
+    -C {{ openshift_node_generated_config_dir }} .
   args:
-    creates: "{{ openshift_generated_configs_dir }}/node-{{ item.openshift.common.hostname }}/server.crt"
-  with_items: "{{ nodes_needing_certs | default([]) }}"
+    creates: "{{ openshift_node_generated_config_dir }}.tgz"
+  when: node_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Retrieve the node config tarballs from the master
+  fetch:
+    src: "{{ openshift_node_generated_config_dir }}.tgz"
+    dest: "{{ node_cert_mktemp.stdout }}/"
+    flat: yes
+    fail_on_missing: yes
+    validate_checksum: yes
+  when: node_certs_missing | bool
+  delegate_to: "{{ openshift_ca_host }}"
+
+- name: Ensure certificate directory exists
+  file:
+    path: "{{ openshift_node_cert_dir }}"
+    state: directory
+  when: node_certs_missing | bool
+
+- name: Unarchive the tarball on the node
+  unarchive:
+    src: "{{ node_cert_mktemp.stdout }}/{{ openshift_node_cert_subdir }}.tgz"
+    dest: "{{ openshift_node_cert_dir }}"
+  when: node_certs_missing | bool
diff --git a/roles/openshift_node_certificates/vars/main.yml b/roles/openshift_node_certificates/vars/main.yml
index 61fbb1e51..2fafc7387 100644
--- a/roles/openshift_node_certificates/vars/main.yml
+++ b/roles/openshift_node_certificates/vars/main.yml
@@ -1,7 +1,6 @@
 ---
-openshift_node_config_dir: "{{ openshift.common.config_base }}/node"
-openshift_master_config_dir: "{{ openshift.common.config_base }}/master"
 openshift_generated_configs_dir: "{{ openshift.common.config_base }}/generated-configs"
-openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
-openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
-openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"
+openshift_node_cert_dir: "{{ openshift.common.config_base }}/node"
+openshift_node_cert_subdir: "node-{{ openshift.common.hostname }}"
+openshift_node_config_dir: "{{ openshift.common.config_base }}/node"
+openshift_node_generated_config_dir: "{{ openshift_generated_configs_dir }}/{{ openshift_node_cert_subdir }}"
diff --git a/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh b/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
index 0d7941e4c..51a43d113 100755
--- a/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
+++ b/roles/openshift_node_dnsmasq/files/networkmanager/99-origin-dns.sh
@@ -51,6 +51,9 @@ EOF
     done
     systemctl restart dnsmasq
 
-    sed -i 's/^nameserver.*$/nameserver '"${def_route_ip}"' # updated by \/etc\/NetworkManager\/dispatcher.d\/99-origin-dns.sh/g' /etc/resolv.conf
+    sed -i 's/^nameserver.*$/nameserver '"${def_route_ip}"'/g' /etc/resolv.conf
+    if ! grep -q '99-origin-dns.sh' /etc/resolv.conf; then
+      echo "# nameserver updated by /etc/NetworkManager/dispatcher.d/99-origin-dns.sh" >> /etc/resolv.conf
+    fi
   fi
 fi