Implement container runtime role

19 files changed, 1097 insertions, 0 deletions

diff --git a/roles/container_runtime/README.md b/roles/container_runtime/README.md
new file mode 100644
index 000000000..e363c1714
--- /dev/null
+++ b/roles/container_runtime/README.md
@@ -0,0 +1,39 @@
+Docker
+=========
+
+Ensures docker package or system container is installed, and optionally raises timeout for systemd-udevd.service to 5 minutes.
+
+container-daemon.json items may be found at https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file
+
+Requirements
+------------
+
+Ansible 2.2
+
+Mandator Role Variables
+--------------
+
+
+
+Dependencies
+------------
+
+Depends on openshift_facts having already been run.
+
+Example Playbook
+----------------
+
+    - hosts: servers
+      roles:
+      - role: container_runtime
+        docker_udev_workaround: "true"
+
+License
+-------
+
+ASL 2.0
+
+Author Information
+------------------
+
+OpenShift operations, Red Hat, Inc
diff --git a/roles/container_runtime/defaults/main.yml b/roles/container_runtime/defaults/main.yml
new file mode 100644
index 000000000..62b3e141a
--- /dev/null
+++ b/roles/container_runtime/defaults/main.yml
@@ -0,0 +1,80 @@
+---
+docker_cli_auth_config_path: '/root/.docker'
+openshift_docker_signature_verification: False
+
+repoquery_cmd: "{{ 'dnf repoquery --latest-limit 1 -d 0' if ansible_pkg_mgr == 'dnf' else 'repoquery --plugins' }}"
+
+openshift_docker_alternative_creds: False
+
+# oreg_url is defined by user input.
+oreg_host: "{{ oreg_url.split('/')[0] if (oreg_url is defined and '.' in oreg_url.split('/')[0]) else '' }}"
+oreg_auth_credentials_replace: False
+
+openshift_docker_use_system_container: False
+openshift_docker_disable_push_dockerhub: False  # bool
+openshift_docker_selinux_enabled: True
+openshift_docker_service_name: "{{ 'container-engine' if (openshift_docker_use_system_container | default(False)) else 'docker' }}"
+
+openshift_docker_hosted_registry_insecure: False  # bool
+
+openshift_docker_hosted_registry_network_default: "{{ openshift_portal_net | default(False) }}"
+openshift_docker_hosted_registry_network: "{{ openshift_docker_hosted_registry_network_default }}"
+
+openshift_docker_additional_registries: []
+openshift_docker_blocked_registries: []
+openshift_docker_insecure_registries: []
+
+openshift_docker_ent_reg: 'registry.access.redhat.com'
+
+openshift_docker_options: False  # str
+openshift_docker_log_driver: False  # str
+openshift_docker_log_options: []
+
+# The l2_docker_* variables convert csv strings to lists, if
+# necessary.  These variables should be used in place of their respective
+# openshift_docker_* counterparts to ensure the properly formatted lists are
+# utilized.
+l2_docker_additional_registries: "{% if openshift_docker_additional_registries is string %}{% if openshift_docker_additional_registries == '' %}[]{% elif ',' in openshift_docker_additional_registries %}{{ openshift_docker_additional_registries.split(',') | list }}{% else %}{{ [ openshift_docker_additional_registries ] }}{% endif %}{% else %}{{ openshift_docker_additional_registries }}{% endif %}"
+l2_docker_blocked_registries: "{% if openshift_docker_blocked_registries is string %}{% if openshift_docker_blocked_registries == '' %}[]{% elif ',' in openshift_docker_blocked_registries %}{{ openshift_docker_blocked_registries.split(',') | list }}{% else %}{{ [ openshift_docker_blocked_registries ] }}{% endif %}{% else %}{{ openshift_docker_blocked_registries }}{% endif %}"
+l2_docker_insecure_registries: "{% if openshift_docker_insecure_registries is string %}{% if openshift_docker_insecure_registries == '' %}[]{% elif ',' in openshift_docker_insecure_registries %}{{ openshift_docker_insecure_registries.split(',') | list }}{% else %}{{ [ openshift_docker_insecure_registries ] }}{% endif %}{% else %}{{ openshift_docker_insecure_registries }}{% endif %}"
+l2_docker_log_options: "{% if openshift_docker_log_options is string %}{% if ',' in openshift_docker_log_options %}{{ openshift_docker_log_options.split(',') | list }}{% else %}{{ [ openshift_docker_log_options ] }}{% endif %}{% else %}{{ openshift_docker_log_options }}{% endif %}"
+
+openshift_docker_use_etc_containers: False
+containers_registries_conf_path: /etc/containers/registries.conf
+
+r_crio_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
+r_crio_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
+
+r_crio_os_firewall_deny: []
+r_crio_os_firewall_allow:
+- service: crio
+  port: 10010/tcp
+
+
+openshift_docker_is_node_or_master: "{{ True if inventory_hostname in (groups['oo_masters_to_config']|default([])) or inventory_hostname in (groups['oo_nodes_to_config']|default([])) else False | bool }}"
+
+docker_alt_storage_path: /var/lib/containers/docker
+docker_default_storage_path: /var/lib/docker
+
+# Set local versions of facts that must be in json format for container-daemon.json
+# NOTE: When jinja2.9+ is used the container-daemon.json file can move to using tojson
+l_docker_log_options: "{{ l2_docker_log_options | to_json }}"
+l_docker_additional_registries: "{{ l2_docker_additional_registries | to_json }}"
+l_docker_blocked_registries: "{{ l2_docker_blocked_registries | to_json }}"
+l_docker_insecure_registries: "{{ l2_docker_insecure_registries | to_json }}"
+l_docker_selinux_enabled: "{{ openshift_docker_selinux_enabled | to_json }}"
+
+docker_http_proxy: "{{ openshift_http_proxy | default('') }}"
+docker_https_proxy: "{{ openshift.common.https_proxy | default('') }}"
+docker_no_proxy: "{{ openshift.common.no_proxy | default('') }}"
+
+openshift_use_crio: False
+openshift_use_crio_only: False
+
+
+l_insecure_crio_registries: "{{ '\"{}\"'.format('\", \"'.join(l2_docker_insecure_registries)) }}"
+l_crio_registries: "{{ l2_docker_additional_registries + ['docker.io'] }}"
+l_additional_crio_registries: "{{ '\"{}\"'.format('\", \"'.join(l_crio_registries)) }}"
+
+l_openshift_image_tag_default: "{{ openshift_release }}"
+l_openshift_image_tag: "{{ openshift_image_tag | default(l_openshift_image_tag_default) | string}}"
diff --git a/roles/container_runtime/handlers/main.yml b/roles/container_runtime/handlers/main.yml
new file mode 100644
index 000000000..67cd6d782
--- /dev/null
+++ b/roles/container_runtime/handlers/main.yml
@@ -0,0 +1,18 @@
+---
+
+- name: restart container runtime
+  systemd:
+    name: "{{ openshift_docker_service_name }}"
+    state: restarted
+    daemon_reload: yes
+  register: r_docker_restart_docker_result
+  until: not r_docker_restart_docker_result | failed
+  retries: 3
+  delay: 30
+  when: not docker_service_status_changed | default(false) | bool
+
+- name: restart udev
+  systemd:
+    name: systemd-udevd
+    state: restarted
+    daemon_reload: yes
diff --git a/roles/container_runtime/meta/main.yml b/roles/container_runtime/meta/main.yml
new file mode 100644
index 000000000..02fceb745
--- /dev/null
+++ b/roles/container_runtime/meta/main.yml
@@ -0,0 +1,15 @@
+---
+galaxy_info:
+  author: OpenShift
+  description: container runtime install and configure
+  company: Red Hat, Inc
+  license: ASL 2.0
+  min_ansible_version: 2.2
+  platforms:
+  - name: EL
+    versions:
+    - 7
+dependencies:
+- role: lib_openshift
+- role: lib_os_firewall
+- role: lib_utils
diff --git a/roles/container_runtime/tasks/crio_firewall.yml b/roles/container_runtime/tasks/crio_firewall.yml
new file mode 100644
index 000000000..fbd1ff515
--- /dev/null
+++ b/roles/container_runtime/tasks/crio_firewall.yml
@@ -0,0 +1,40 @@
+---
+- when: r_crio_firewall_enabled | bool and not r_crio_use_firewalld | bool
+  block:
+  - name: Add iptables allow rules
+    os_firewall_manage_iptables:
+      name: "{{ item.service }}"
+      action: add
+      protocol: "{{ item.port.split('/')[1] }}"
+      port: "{{ item.port.split('/')[0] }}"
+    when: item.cond | default(True)
+    with_items: "{{ r_crio_os_firewall_allow }}"
+
+  - name: Remove iptables rules
+    os_firewall_manage_iptables:
+      name: "{{ item.service }}"
+      action: remove
+      protocol: "{{ item.port.split('/')[1] }}"
+      port: "{{ item.port.split('/')[0] }}"
+    when: item.cond | default(True)
+    with_items: "{{ r_crio_os_firewall_deny }}"
+
+- when: r_crio_firewall_enabled | bool and r_crio_use_firewalld | bool
+  block:
+  - name: Add firewalld allow rules
+    firewalld:
+      port: "{{ item.port }}"
+      permanent: true
+      immediate: true
+      state: enabled
+    when: item.cond | default(True)
+    with_items: "{{ r_crio_os_firewall_allow }}"
+
+  - name: Remove firewalld allow rules
+    firewalld:
+      port: "{{ item.port }}"
+      permanent: true
+      immediate: true
+      state: disabled
+    when: item.cond | default(True)
+    with_items: "{{ r_crio_os_firewall_deny }}"
diff --git a/roles/container_runtime/tasks/main.yml b/roles/container_runtime/tasks/main.yml
new file mode 100644
index 000000000..6d68082b1
--- /dev/null
+++ b/roles/container_runtime/tasks/main.yml
@@ -0,0 +1,85 @@
+---
+- include_tasks: udev_workaround.yml
+  when: docker_udev_workaround | default(False) | bool
+
+- name: Add enterprise registry, if necessary
+  set_fact:
+    l2_docker_additional_registries: "{{ l2_docker_additional_registries + [openshift_docker_ent_reg] }}"
+  when:
+    - openshift.common.deployment_type == 'openshift-enterprise'
+    - openshift_docker_ent_reg != ''
+    - openshift_docker_ent_reg not in l2_docker_additional_registries
+    - not openshift_use_crio_only | bool
+
+- name: Use Package Docker if Requested
+  include_tasks: package_docker.yml
+  when:
+    - not openshift_docker_use_system_container
+    - not openshift_use_crio_only
+
+- name: Ensure /var/lib/containers exists
+  file:
+    path: /var/lib/containers
+    state: directory
+
+- name: Fix SELinux Permissions on /var/lib/containers
+  command: "restorecon -R /var/lib/containers/"
+  changed_when: false
+
+- name: Use System Container Docker if Requested
+  include_tasks: systemcontainer_docker.yml
+  when:
+    - openshift_docker_use_system_container
+    - not openshift_use_crio_only
+
+- name: Add CRI-O usage Requested
+  include_tasks: systemcontainer_crio.yml
+  when:
+    - openshift_use_crio
+    - openshift_docker_is_node_or_master | bool
+
+- name: stat the docker data dir
+  stat:
+    path: "{{ docker_default_storage_path }}"
+  register: dockerstat
+
+- when:
+    - openshift_use_crio
+    - dockerstat.stat.islnk is defined and not (dockerstat.stat.islnk | bool)
+  block:
+    - name: stop the current running docker
+      systemd:
+        state: stopped
+        name: "{{ openshift_docker_service_name }}"
+
+    - name: copy "{{ docker_default_storage_path }}" to "{{ docker_alt_storage_path }}"
+      command: "cp -r {{ docker_default_storage_path }} {{ docker_alt_storage_path }}"
+      register: results
+      failed_when:
+        - results.rc != 0
+
+    - name: "Set the selinux context on {{ docker_alt_storage_path }}"
+      command: "semanage fcontext -a -e {{ docker_default_storage_path }} {{ docker_alt_storage_path }}"
+      register: results
+      failed_when:
+        - results.rc == 1
+        - "'already exists' not in results.stderr"
+
+    - name: "restorecon the {{ docker_alt_storage_path }}"
+      command: "restorecon -r {{ docker_alt_storage_path }}"
+
+    - name: Remove the old docker location
+      file:
+        state: absent
+        path: "{{ docker_default_storage_path }}"
+
+    - name: Setup the link
+      file:
+        state: link
+        src: "{{ docker_alt_storage_path }}"
+        path: "{{ docker_default_storage_path }}"
+
+    - name: start docker
+      systemd:
+        state: started
+        name: "{{ openshift_docker_service_name }}"
diff --git a/roles/container_runtime/tasks/package_docker.yml b/roles/container_runtime/tasks/package_docker.yml
new file mode 100644
index 000000000..40ab75a25
--- /dev/null
+++ b/roles/container_runtime/tasks/package_docker.yml
@@ -0,0 +1,167 @@
+---
+- name: Get current installed Docker version
+  command: "{{ repoquery_installed }} --qf '%{version}' docker"
+  when: not openshift.common.is_atomic | bool
+  register: curr_docker_version
+  retries: 4
+  until: curr_docker_version | succeeded
+  changed_when: false
+
+- name: Error out if Docker pre-installed but too old
+  fail:
+    msg: "Docker {{ curr_docker_version.stdout }} is installed, but >= 1.9.1 is required."
+  when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.9.1', '<') and not docker_version is defined
+
+- name: Error out if requested Docker is too old
+  fail:
+    msg: "Docker {{ docker_version }} requested, but >= 1.9.1 is required."
+  when: docker_version is defined and docker_version | version_compare('1.9.1', '<')
+
+# If a docker_version was requested, sanity check that we can install or upgrade to it, and
+# no downgrade is required.
+- name: Fail if Docker version requested but downgrade is required
+  fail:
+    msg: "Docker {{ curr_docker_version.stdout }} is installed, but version {{ docker_version }} was requested."
+  when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and docker_version is defined and curr_docker_version.stdout | version_compare(docker_version, '>')
+
+# This involves an extremely slow migration process, users should instead run the
+# Docker 1.10 upgrade playbook to accomplish this.
+- name: Error out if attempting to upgrade Docker across the 1.10 boundary
+  fail:
+    msg: "Cannot upgrade Docker to >= 1.10, please upgrade or remove Docker manually, or use the Docker upgrade playbook if OpenShift is already installed."
+  when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.10', '<') and docker_version is defined and docker_version | version_compare('1.10', '>=')
+
+# Make sure Docker is installed, but does not update a running version.
+# Docker upgrades are handled by a separate playbook.
+# Note: The curr_docker_version.stdout check can be removed when https://github.com/ansible/ansible/issues/33187 gets fixed.
+- name: Install Docker
+  package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
+  when: not openshift.common.is_atomic | bool and not curr_docker_version | skipped and not curr_docker_version.stdout != ''
+  register: result
+  until: result | success
+
+- block:
+  # Extend the default Docker service unit file when using iptables-services
+  - name: Ensure docker.service.d directory exists
+    file:
+      path: "{{ docker_systemd_dir }}"
+      state: directory
+
+  - name: Configure Docker service unit file
+    template:
+      dest: "{{ docker_systemd_dir }}/custom.conf"
+      src: custom.conf.j2
+    notify:
+    - restart container runtime
+  when: not (os_firewall_use_firewalld | default(False)) | bool
+
+- stat: path=/etc/sysconfig/docker
+  register: docker_check
+
+- name: Set registry params
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^{{ item.reg_conf_var }}=.*$'
+    line: "{{ item.reg_conf_var }}='{{ item.reg_fact_val | oo_prepend_strings_in_list(item.reg_flag ~ ' ') | join(' ') }}'"
+  when:
+  - item.reg_fact_val != []
+  - docker_check.stat.isreg is defined
+  - docker_check.stat.isreg
+  with_items:
+  - reg_conf_var: ADD_REGISTRY
+    reg_fact_val: "{{ l2_docker_additional_registries }}"
+    reg_flag: --add-registry
+  - reg_conf_var: BLOCK_REGISTRY
+    reg_fact_val: "{{ l2_docker_blocked_registries }}"
+    reg_flag: --block-registry
+  - reg_conf_var: INSECURE_REGISTRY
+    reg_fact_val: "{{ l2_docker_insecure_registries }}"
+    reg_flag: --insecure-registry
+  notify:
+  - restart container runtime
+
+- name: Place additional/blocked/insecure registries in /etc/containers/registries.conf
+  template:
+    dest: "{{ containers_registries_conf_path }}"
+    src: registries.conf
+  when: openshift_docker_use_etc_containers | bool
+  notify:
+  - restart container runtime
+
+- name: Set Proxy Settings
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^{{ item.reg_conf_var }}=.*$'
+    line: "{{ item.reg_conf_var }}='{{ item.reg_fact_val }}'"
+    state: "{{ 'present' if item.reg_fact_val != '' else 'absent'}}"
+  with_items:
+  - reg_conf_var: HTTP_PROXY
+    reg_fact_val: "{{ docker_http_proxy }}"
+  - reg_conf_var: HTTPS_PROXY
+    reg_fact_val: "{{ docker_https_proxy }}"
+  - reg_conf_var: NO_PROXY
+    reg_fact_val: "{{ docker_no_proxy }}"
+  notify:
+  - restart container runtime
+  when:
+  - docker_check.stat.isreg is defined
+  - docker_check.stat.isreg
+  - docker_http_proxy != '' or docker_https_proxy != ''
+
+- name: Set various Docker options
+  lineinfile:
+    dest: /etc/sysconfig/docker
+    regexp: '^OPTIONS=.*$'
+    line: "OPTIONS='\
+      {% if ansible_selinux.status | default(None) == 'enabled' and openshift_docker_selinux_enabled | default(true) | bool %} --selinux-enabled {% endif %} \
+      {% if openshift_docker_log_driver | bool %} --log-driver {{ openshift_docker_log_driver }}{% endif %} \
+      {% if l2_docker_log_options != [] %} {{ l2_docker_log_options |  oo_split() | oo_prepend_strings_in_list('--log-opt ') | join(' ')}}{% endif %} \
+      {% if openshift_docker_hosted_registry_insecure and (openshift_docker_hosted_registry_network | bool) %} --insecure-registry={{ openshift_docker_hosted_registry_network }} {% endif %} \
+      {% if docker_options is defined %} {{ docker_options }}{% endif %} \
+      {% if openshift_docker_options %} {{ openshift_docker_options }}{% endif %} \
+      {% if openshift_docker_disable_push_dockerhub %} --confirm-def-push={{ openshift_docker_disable_push_dockerhub | bool }}{% endif %} \
+      --signature-verification={{ openshift_docker_signature_verification | bool }}'"
+  when: docker_check.stat.isreg is defined and docker_check.stat.isreg
+  notify:
+  - restart container runtime
+
+- stat: path=/etc/sysconfig/docker-network
+  register: sysconfig_docker_network_check
+
+- name: Configure Docker Network OPTIONS
+  lineinfile:
+    dest: /etc/sysconfig/docker-network
+    regexp: '^DOCKER_NETWORK_OPTIONS=.*$'
+    line: "DOCKER_NETWORK_OPTIONS='\
+      {% if openshift.node is defined and openshift.node.sdn_mtu is defined %} --mtu={{ openshift.node.sdn_mtu }}{% endif %}'"
+  when:
+  - sysconfig_docker_network_check.stat.isreg is defined
+  - sysconfig_docker_network_check.stat.isreg
+  notify:
+  - restart container runtime
+
+# The following task is needed as the systemd module may report a change in
+# state even though docker is already running.
+- name: Detect if docker is already started
+  command: "systemctl show docker -p ActiveState"
+  changed_when: False
+  register: r_docker_already_running_result
+
+- name: Start the Docker service
+  systemd:
+    name: docker
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  register: r_docker_package_docker_start_result
+  until: not r_docker_package_docker_start_result | failed
+  retries: 3
+  delay: 30
+
+- set_fact:
+    docker_service_status_changed: "{{ (r_docker_package_docker_start_result | changed) and (r_docker_already_running_result.stdout != 'ActiveState=active' ) }}"
+
+- meta: flush_handlers
+
+# This needs to run after docker is restarted to account for proxy settings.
+- include_tasks: registry_auth.yml
diff --git a/roles/container_runtime/tasks/registry_auth.yml b/roles/container_runtime/tasks/registry_auth.yml
new file mode 100644
index 000000000..2c7bc5711
--- /dev/null
+++ b/roles/container_runtime/tasks/registry_auth.yml
@@ -0,0 +1,32 @@
+---
+- name: Check for credentials file for registry auth
+  stat:
+    path: "{{ docker_cli_auth_config_path }}/config.json"
+  when: oreg_auth_user is defined
+  register: docker_cli_auth_credentials_stat
+
+- name: Create credentials for docker cli registry auth
+  command: "docker --config={{ docker_cli_auth_config_path }} login -u {{ oreg_auth_user }} -p {{ oreg_auth_password }} {{ oreg_host }}"
+  register: openshift_docker_credentials_create_res
+  retries: 3
+  delay: 5
+  until: openshift_docker_credentials_create_res.rc == 0
+  when:
+  - not openshift_docker_alternative_creds | bool
+  - oreg_auth_user is defined
+  - (not docker_cli_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool
+
+# docker_creds is a custom module from lib_utils
+# 'docker login' requires a docker.service running on the local host, this is an
+# alternative implementation for non-docker hosts.  This implementation does not
+# check the registry to determine whether or not the credentials will work.
+- name: Create credentials for docker cli registry auth (alternative)
+  docker_creds:
+    path: "{{ docker_cli_auth_config_path }}"
+    registry: "{{ oreg_host }}"
+    username: "{{ oreg_auth_user }}"
+    password: "{{ oreg_auth_password }}"
+  when:
+  - openshift_docker_alternative_creds | bool
+  - oreg_auth_user is defined
+  - (not docker_cli_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool
diff --git a/roles/container_runtime/tasks/systemcontainer_crio.yml b/roles/container_runtime/tasks/systemcontainer_crio.yml
new file mode 100644
index 000000000..8dcfe60ef
--- /dev/null
+++ b/roles/container_runtime/tasks/systemcontainer_crio.yml
@@ -0,0 +1,164 @@
+---
+# TODO: Much of this file is shared with container engine tasks
+
+- name: Ensure container-selinux is installed
+  package:
+    name: container-selinux
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+- name: Check we are not using node as a Docker container with CRI-O
+  fail: msg='Cannot use CRI-O with node configured as a Docker container'
+  when:
+    - openshift.common.is_containerized | bool
+    - not openshift.common.is_node_system_container | bool
+
+# Used to pull and install the system container
+- name: Ensure atomic is installed
+  package:
+    name: atomic
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+# At the time of writing the atomic command requires runc for it's own use. This
+# task is here in the even that the atomic package ever removes the dependency.
+- name: Ensure runc is installed
+  package:
+    name: runc
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+
+- name: Check that overlay is in the kernel
+  shell: lsmod | grep overlay
+  register: l_has_overlay_in_kernel
+  ignore_errors: yes
+  failed_when: false
+
+- when: l_has_overlay_in_kernel.rc != 0
+  block:
+
+    - name: Add overlay to modprobe.d
+      template:
+        dest: /etc/modules-load.d/overlay.conf
+        src: overlay.conf.j2
+        backup: yes
+
+    - name: Manually modprobe overlay into the kernel
+      command: modprobe overlay
+
+    - name: Enable and start systemd-modules-load
+      service:
+        name: systemd-modules-load
+        enabled: yes
+        state: restarted
+
+- name: Ensure proxies are in the atomic.conf
+  include_role:
+    name: openshift_atomic
+    tasks_from: proxy
+
+- block:
+
+    - name: Set CRI-O image defaults
+      set_fact:
+        l_crio_image_prepend: "docker.io/gscrivano"
+        l_crio_image_name: "cri-o-fedora"
+        l_crio_image_tag: "latest"
+
+    - name: Use Centos based image when distribution is CentOS
+      set_fact:
+        l_crio_image_name: "cri-o-centos"
+      when: ansible_distribution == "CentOS"
+
+    - name: Set CRI-O image tag
+      set_fact:
+        l_crio_image_tag: "{{ l_openshift_image_tag }}"
+      when:
+        - openshift_deployment_type == 'openshift-enterprise'
+
+    - name: Use RHEL based image when distribution is Red Hat
+      set_fact:
+        l_crio_image_prepend: "registry.access.redhat.com/openshift3"
+        l_crio_image_name: "cri-o"
+      when: ansible_distribution == "RedHat"
+
+    - name: Set the full image name
+      set_fact:
+        l_crio_image: "{{ l_crio_image_prepend }}/{{ l_crio_image_name }}:{{ l_crio_image_tag }}"
+
+    # For https://github.com/openshift/aos-cd-jobs/pull/624#pullrequestreview-61816548
+    - name: Use a specific image if requested
+      set_fact:
+        l_crio_image: "{{ openshift_crio_systemcontainer_image_override }}"
+      when:
+        - openshift_crio_systemcontainer_image_override is defined
+        - openshift_crio_systemcontainer_image_override != ""
+
+    # Be nice and let the user see the variable result
+    - debug:
+        var: l_crio_image
+
+# NOTE: no_proxy added as a workaround until https://github.com/projectatomic/atomic/pull/999 is released
+- name: Pre-pull CRI-O System Container image
+  command: "atomic pull --storage ostree {{ l_crio_image }}"
+  changed_when: false
+  environment:
+    NO_PROXY: "{{ openshift.common.no_proxy | default('') }}"
+
+
+- name: Install CRI-O System Container
+  oc_atomic_container:
+    name: "cri-o"
+    image: "{{ l_crio_image }}"
+    state: latest
+
+- name: Remove CRI-O default configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/cni/net.d/200-loopback.conf
+    - /etc/cni/net.d/100-crio-bridge.conf
+
+- name: Create the CRI-O configuration
+  template:
+    dest: /etc/crio/crio.conf
+    src: crio.conf.j2
+    backup: yes
+
+- name: Ensure CNI configuration directory exists
+  file:
+    path: /etc/cni/net.d/
+    state: directory
+
+- name: setup firewall for CRI-O
+  include_tasks: crio_firewall.yml
+  static: yes
+
+- name: Configure the CNI network
+  template:
+    dest: /etc/cni/net.d/openshift-sdn.conf
+    src: 80-openshift-sdn.conf.j2
+
+- name: Start the CRI-O service
+  systemd:
+    name: "cri-o"
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  register: start_result
+
+- meta: flush_handlers
+
+# If we are using crio only, docker.service might not be available for
+# 'docker login'
+- include_tasks: registry_auth.yml
+  vars:
+    openshift_docker_alternative_creds: "{{ openshift_use_crio_only }}"
diff --git a/roles/container_runtime/tasks/systemcontainer_docker.yml b/roles/container_runtime/tasks/systemcontainer_docker.yml
new file mode 100644
index 000000000..84217e50c
--- /dev/null
+++ b/roles/container_runtime/tasks/systemcontainer_docker.yml
@@ -0,0 +1,163 @@
+---
+# If docker_options are provided we should fail. We should not install docker and ignore
+# the users configuration. NOTE: docker_options == inventory:openshift_docker_options
+- name: Fail quickly if openshift_docker_options are set
+  assert:
+    that:
+      - "{% if not openshift_docker_options %}1{% else %}0{% endif %}"
+    msg: |
+      Docker via System Container does not allow for the use of the openshift_docker_options
+      variable. If you want to use openshift_docker_options you will need to use the
+      traditional docker package install. Otherwise, comment out openshift_docker_options
+      in your inventory file.
+
+- name: Ensure container-selinux is installed
+  package:
+    name: container-selinux
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+# Used to pull and install the system container
+- name: Ensure atomic is installed
+  package:
+    name: atomic
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+# At the time of writing the atomic command requires runc for it's own use. This
+# task is here in the even that the atomic package ever removes the dependency.
+- name: Ensure runc is installed
+  package:
+    name: runc
+    state: present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+# Make sure Docker is installed so we are able to use the client
+- name: Install Docker so we can use the client
+  package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present
+  when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
+
+# Make sure docker is disabled. Errors are ignored.
+- name: Disable Docker
+  systemd:
+    name: docker
+    enabled: no
+    state: stopped
+    daemon_reload: yes
+  ignore_errors: True
+  register: r_docker_systemcontainer_docker_stop_result
+  until: not r_docker_systemcontainer_docker_stop_result | failed
+  retries: 3
+  delay: 30
+
+- name: Ensure proxies are in the atomic.conf
+  include_role:
+    name: openshift_atomic
+    tasks_from: proxy
+
+- block:
+
+    - name: Set to default prepend
+      set_fact:
+        l_docker_image_prepend: "gscrivano"
+        l_docker_image_tag: "latest"
+
+    - name: Set container engine image tag
+      set_fact:
+        l_docker_image_tag: "{{ l_openshift_image_tag }}"
+      when:
+        - openshift_deployment_type == 'openshift-enterprise'
+
+    - name: Use Red Hat Registry for image when distribution is Red Hat
+      set_fact:
+        l_docker_image_prepend: "registry.access.redhat.com/openshift3"
+      when: ansible_distribution == 'RedHat'
+
+    - name: Use Fedora Registry for image when distribution is Fedora
+      set_fact:
+        l_docker_image_prepend: "registry.fedoraproject.org/f25"
+      when: ansible_distribution == 'Fedora'
+
+    - name: Set the full image name
+      set_fact:
+        l_docker_image: "{{ l_docker_image_prepend }}/{{ openshift_docker_service_name }}:{{ l_docker_image_tag }}"
+
+    # For https://github.com/openshift/openshift-ansible/pull/5354#issuecomment-328552959
+    - name: Use a specific image if requested
+      set_fact:
+        l_docker_image: "{{ openshift_docker_systemcontainer_image_override }}"
+      when:
+        - openshift_docker_systemcontainer_image_override is defined
+        - openshift_docker_systemcontainer_image_override != ""
+
+    # Be nice and let the user see the variable result
+    - debug:
+        var: l_docker_image
+
+# NOTE: no_proxy added as a workaround until https://github.com/projectatomic/atomic/pull/999 is released
+- name: Pre-pull Container Engine System Container image
+  command: "atomic pull --storage ostree {{ l_docker_image }}"
+  changed_when: false
+  environment:
+    NO_PROXY: "{{ docker_no_proxy }}"
+
+
+- name: Ensure container-engine.service.d directory exists
+  file:
+    path: "{{ container_engine_systemd_dir }}"
+    state: directory
+
+- name: Ensure /etc/docker directory exists
+  file:
+    path: "{{ docker_conf_dir }}"
+    state: directory
+
+- name: Install Container Engine System Container
+  oc_atomic_container:
+    name: "{{ openshift_docker_service_name }}"
+    image: "{{ l_docker_image }}"
+    state: latest
+
+- name: Configure Container Engine Service File
+  template:
+    dest: "{{ container_engine_systemd_dir }}/custom.conf"
+    src: systemcontainercustom.conf.j2
+
+# Configure container-engine using the container-daemon.json file
+# NOTE: daemon.json and container-daemon.json have been seperated to avoid
+#       collision.
+- name: Configure Container Engine
+  template:
+    dest: "{{ docker_conf_dir }}/container-daemon.json"
+    src: daemon.json
+
+# Enable and start the container-engine service
+- name: Start the Container Engine service
+  systemd:
+    name: "{{ openshift_docker_service_name }}"
+    enabled: yes
+    state: started
+    daemon_reload: yes
+  register: r_docker_systemcontainer_docker_start_result
+  until: not r_docker_systemcontainer_docker_start_result | failed
+  retries: 3
+  delay: 30
+
+- set_fact:
+    docker_service_status_changed: "{{ r_docker_systemcontainer_docker_start_result | changed }}"
+
+- meta: flush_handlers
+
+# Since docker is running as a system container, docker login will fail to create
+# credentials.  Use alternate method if requiring authenticated registries.
+- include_tasks: registry_auth.yml
+  vars:
+    openshift_docker_alternative_creds: True
diff --git a/roles/container_runtime/tasks/udev_workaround.yml b/roles/container_runtime/tasks/udev_workaround.yml
new file mode 100644
index 000000000..257c3123d
--- /dev/null
+++ b/roles/container_runtime/tasks/udev_workaround.yml
@@ -0,0 +1,24 @@
+---
+
+- name: Getting current systemd-udevd exec command
+  command: grep -e "^ExecStart=" /lib/systemd/system/systemd-udevd.service
+  changed_when: false
+  register: udevw_udev_start_cmd
+
+- name: Assure systemd-udevd.service.d directory exists
+  file:
+    path: "{{ udevw_udevd_dir }}"
+    state: directory
+
+- name: Create systemd-udevd override file
+  copy:
+    content: |
+      [Service]
+      #Need blank ExecStart to "clear" pre-existing one
+      ExecStart=
+      {{ udevw_udev_start_cmd.stdout }} --event-timeout=300
+    dest: "{{ udevw_udevd_dir }}/override.conf"
+    owner: root
+    mode: "0644"
+  notify:
+    - restart udev
diff --git a/roles/container_runtime/templates/80-openshift-sdn.conf.j2 b/roles/container_runtime/templates/80-openshift-sdn.conf.j2
new file mode 100644
index 000000000..a693aea5f
--- /dev/null
+++ b/roles/container_runtime/templates/80-openshift-sdn.conf.j2
@@ -0,0 +1,5 @@
+{
+  "cniVersion": "0.1.0",
+  "name": "openshift-sdn",
+  "type": "openshift-sdn"
+}
diff --git a/roles/container_runtime/templates/crio.conf.j2 b/roles/container_runtime/templates/crio.conf.j2
new file mode 100644
index 000000000..3f066a17f
--- /dev/null
+++ b/roles/container_runtime/templates/crio.conf.j2
@@ -0,0 +1,164 @@
+# {{ ansible_managed }}
+
+# The "crio" table contains all of the server options.
+[crio]
+
+# root is a path to the "root directory". CRIO stores all of its data,
+# including container images, in this directory.
+root = "/var/lib/containers/storage"
+
+# run is a path to the "run directory". CRIO stores all of its state
+# in this directory.
+runroot = "/var/run/containers/storage"
+
+# storage_driver select which storage driver is used to manage storage
+# of images and containers.
+storage_driver = "overlay"
+
+# storage_option is used to pass an option to the storage driver.
+storage_option = [
+{% if ansible_distribution in ['RedHat', 'CentOS'] %}
+	"overlay.override_kernel_check=1"
+{% endif %}
+]
+
+# The "crio.api" table contains settings for the kubelet/gRPC
+# interface (which is also used by crioctl).
+[crio.api]
+
+# listen is the path to the AF_LOCAL socket on which crio will listen.
+listen = "/var/run/crio.sock"
+
+# stream_address is the IP address on which the stream server will listen
+stream_address = ""
+
+# stream_port is the port on which the stream server will listen
+stream_port = "10010"
+
+# file_locking is whether file-based locking will be used instead of
+# in-memory locking
+file_locking = true
+
+# The "crio.runtime" table contains settings pertaining to the OCI
+# runtime used and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "/usr/bin/runc"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = ""
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "trusted"
+
+# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
+no_pivot = false
+
+# conmon is the path to conmon binary, used for managing the runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# conmon_env is the environment variable list for conmon process,
+# used for passing necessary environment variable to conmon or runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# selinux indicates whether or not SELinux will be used for pod
+# separation on the host. If you enable this flag, SELinux must be running
+# on the host.
+selinux = true
+
+# seccomp_profile is the seccomp json profile path which is used as the
+# default for the runtime.
+seccomp_profile = "/etc/crio/seccomp.json"
+
+# apparmor_profile is the apparmor profile name which is used as the
+# default for the runtime.
+apparmor_profile = "crio-default"
+
+# cgroup_manager is the cgroup management implementation to be used
+# for the runtime.
+cgroup_manager = "systemd"
+
+# hooks_dir_path is the oci hooks directory for automatically executed hooks
+hooks_dir_path = "/usr/share/containers/oci/hooks.d"
+
+# default_mounts is the mounts list to be mounted for the container when created
+default_mounts = [
+	"/usr/share/rhel/secrets:/run/secrets",
+]
+
+# pids_limit is the number of processes allowed in a container
+pids_limit = 1024
+
+# log_size_max is the max limit for the container log size in bytes.
+# Negative values indicate that no limit is imposed.
+log_size_max = 52428800
+
+# The "crio.image" table contains settings pertaining to the
+# management of OCI images.
+[crio.image]
+
+# default_transport is the prefix we try prepending to an image name if the
+# image name as we receive it can't be parsed as a valid source reference
+default_transport = "docker://"
+
+# pause_image is the image which we use to instantiate infra containers.
+pause_image = "kubernetes/pause"
+
+# pause_command is the command to run in a pause_image to have a container just
+# sit there.  If the image contains the necessary information, this value need
+# not be specified.
+pause_command = "/pause"
+
+# signature_policy is the name of the file which decides what sort of policy we
+# use when deciding whether or not to trust an image that we've pulled.
+# Outside of testing situations, it is strongly advised that this be left
+# unspecified so that the default system-wide policy will be used.
+signature_policy = ""
+
+# image_volumes controls how image volumes are handled.
+# The valid values are mkdir and ignore.
+image_volumes = "mkdir"
+
+# insecure_registries is used to skip TLS verification when pulling images.
+insecure_registries = [
+{{ l_insecure_crio_registries|default("") }}
+]
+
+# registries is used to specify a comma separated list of registries to be used
+# when pulling an unqualified image (e.g. fedora:rawhide).
+registries = [
+{{ l_additional_crio_registries|default("") }}
+]
+
+# The "crio.network" table contains settings pertaining to the
+# management of CNI plugins.
+[crio.network]
+
+# network_dir is is where CNI network configuration
+# files are stored.
+network_dir = "/etc/cni/net.d/"
+
+# plugin_dir is is where CNI plugin binaries are stored.
+plugin_dir = "/opt/cni/bin/"
diff --git a/roles/container_runtime/templates/custom.conf.j2 b/roles/container_runtime/templates/custom.conf.j2
new file mode 100644
index 000000000..713412473
--- /dev/null
+++ b/roles/container_runtime/templates/custom.conf.j2
@@ -0,0 +1,11 @@
+# {{ ansible_managed }}
+
+[Unit]
+Wants=iptables.service
+After=iptables.service
+
+# The following line is a work-around to ensure docker is restarted whenever
+# iptables is restarted.  This ensures the proper iptables rules will be in
+# place for docker.
+# Note:  This will also cause docker to be stopped if iptables is stopped.
+PartOf=iptables.service
diff --git a/roles/container_runtime/templates/daemon.json b/roles/container_runtime/templates/daemon.json
new file mode 100644
index 000000000..383963bd3
--- /dev/null
+++ b/roles/container_runtime/templates/daemon.json
@@ -0,0 +1,20 @@
+{
+    "authorization-plugins": ["rhel-push-plugin"],
+    "default-runtime": "oci",
+    "containerd": "/run/containerd.sock",
+    "disable-legacy-registry": false,
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "insecure-registries": {{ l_docker_insecure_registries }},
+{% if openshift_docker_log_driver is defined  %}
+    "log-driver": "{{ openshift_docker_log_driver }}",
+{%- endif %}
+    "log-opts": {{ l_docker_log_options }},
+    "runtimes": {
+	"oci": {
+	    "path": "/usr/libexec/docker/docker-runc-current"
+	}
+    },
+    "selinux-enabled": {{ l_docker_selinux_enabled | lower }},
+    "add-registry": {{ l_docker_additional_registries }},
+    "block-registry": {{ l_docker_blocked_registries }}
+}
diff --git a/roles/container_runtime/templates/overlay.conf.j2 b/roles/container_runtime/templates/overlay.conf.j2
new file mode 100644
index 000000000..782f46c2e
--- /dev/null
+++ b/roles/container_runtime/templates/overlay.conf.j2
@@ -0,0 +1,2 @@
+### {{ ansible_managed }}
+overlay
diff --git a/roles/container_runtime/templates/registries.conf b/roles/container_runtime/templates/registries.conf
new file mode 100644
index 000000000..d379b2be0
--- /dev/null
+++ b/roles/container_runtime/templates/registries.conf
@@ -0,0 +1,46 @@
+# {{ ansible_managed }}
+# This is a system-wide configuration file used to
+# keep track of registries for various container backends.
+# It adheres to YAML format and does not support recursive
+# lists of registries.
+
+# The default location for this configuration file is /etc/containers/registries.conf.
+
+# The only valid categories are: 'registries', 'insecure_registries',
+# and 'block_registries'.
+
+
+#registries:
+#  - registry.access.redhat.com
+
+{% if l2_docker_additional_registries %}
+registries:
+{% for reg in l2_docker_additional_registries %}
+  - {{ reg }}
+{% endfor %}
+{% endif %}
+
+# If you need to access insecure registries, uncomment the section below
+# and add the registries fully-qualified name. An insecure registry is one
+# that does not have a valid SSL certificate or only does HTTP.
+#insecure_registries:
+#  -
+
+{% if l2_docker_insecure_registries %}
+insecure_registries:
+{% for reg in l2_docker_insecure_registries %}
+  - {{ reg }}
+{% endfor %}
+{% endif %}
+
+# If you need to block pull access from a registry, uncomment the section below
+# and add the registries fully-qualified name.
+#block_registries:
+# -
+
+{% if l2_docker_blocked_registries %}
+block_registries:
+{% for reg in l2_docker_blocked_registries %}
+  - {{ reg }}
+{% endfor %}
+{% endif %}
diff --git a/roles/container_runtime/templates/systemcontainercustom.conf.j2 b/roles/container_runtime/templates/systemcontainercustom.conf.j2
new file mode 100644
index 000000000..86eebfba6
--- /dev/null
+++ b/roles/container_runtime/templates/systemcontainercustom.conf.j2
@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+
+[Service]
+{% if "http_proxy" in openshift.common %}
+Environment=HTTP_PROXY={{ docker_http_proxy }}
+{% endif -%}
+{% if "https_proxy" in openshift.common %}
+Environment=HTTPS_PROXY={{ docker_http_proxy }}
+{% endif -%}
+{% if "no_proxy" in openshift.common %}
+Environment=NO_PROXY={{ docker_no_proxy }}
+{% endif %}
+{%- if os_firewall_use_firewalld|default(false) %}
+[Unit]
+Wants=iptables.service
+After=iptables.service
+{%- endif %}
diff --git a/roles/container_runtime/vars/main.yml b/roles/container_runtime/vars/main.yml
new file mode 100644
index 000000000..4e940b7f5
--- /dev/null
+++ b/roles/container_runtime/vars/main.yml
@@ -0,0 +1,5 @@
+---
+docker_systemd_dir: /etc/systemd/system/docker.service.d
+container_engine_systemd_dir: /etc/systemd/system/container-engine.service.d
+docker_conf_dir: /etc/docker/
+udevw_udevd_dir: /etc/systemd/system/systemd-udevd.service.d