Merge pull request #190 from jwhonce/wip/containers

Infrastructure - Deploy services in cluster

@twiest All comments covered.

29 files changed, 351 insertions, 19 deletions

diff --git a/README_OSE.md b/README_OSE.md
index 41a6f2935..dffabc714 100644
--- a/README_OSE.md
+++ b/README_OSE.md
@@ -80,7 +80,7 @@ ansible_ssh_user=root
 deployment_type=enterprise
 
 # Pre-release registry URL
-openshift_registry_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
+oreg_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
 
 # Pre-release additional repo
 openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel',
diff --git a/inventory/byo/hosts b/inventory/byo/hosts
index 728eec8aa..9a1cbce29 100644
--- a/inventory/byo/hosts
+++ b/inventory/byo/hosts
@@ -17,7 +17,7 @@ ansible_ssh_user=root
 deployment_type=enterprise
 
 # Pre-release registry URL
-openshift_registry_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
+oreg_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
 
 # Pre-release additional repo
 #openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterprise/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}]
diff --git a/playbooks/aws/openshift-cluster/launch.yml b/playbooks/aws/openshift-cluster/launch.yml
index 3eb5496e4..33e1ec25d 100644
--- a/playbooks/aws/openshift-cluster/launch.yml
+++ b/playbooks/aws/openshift-cluster/launch.yml
@@ -25,6 +25,14 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
 
+  - set_fact:
+      a_master: "{{ master_names[0] }}"
+  - add_host: name={{ a_master }} groups=service_master
+
 - include: update.yml
 
+- include: ../../common/openshift-cluster/create_services.yml
+  vars:
+     g_svc_master: "{{ service_master }}"
+
 - include: list.yml
diff --git a/playbooks/common/openshift-cluster/create_services.yml b/playbooks/common/openshift-cluster/create_services.yml
new file mode 100644
index 000000000..e70709d19
--- /dev/null
+++ b/playbooks/common/openshift-cluster/create_services.yml
@@ -0,0 +1,8 @@
+---
+- name: Deploy OpenShift Services
+  hosts: "{{ g_svc_master }}"
+  connection: ssh
+  gather_facts: yes
+  roles:
+  - openshift_registry
+  - openshift_router
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 05822d118..4df64e95f 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -6,6 +6,7 @@
   roles:
   - openshift_master
   - { role: openshift_sdn_master, when: openshift.common.use_openshift_sdn | bool }
+  - { role: fluentd_master, when openshift.common.use_fluentd | bool }
   tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 96641a274..70711e39b 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -96,6 +96,7 @@
   roles:
   - openshift_node
   - { role: openshift_sdn_node, when: openshift.common.use_openshift_sdn | bool }
+  - { role: fluentd_node, when: openshift.common.use_fluentd | bool }
   tasks:
   - name: Create group for deployment type
     group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }}
diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml
index 771f51e91..35737f03d 100644
--- a/playbooks/gce/openshift-cluster/launch.yml
+++ b/playbooks/gce/openshift-cluster/launch.yml
@@ -23,6 +23,22 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
 
+  - set_fact:
+      a_master: "{{ master_names[0] }}"
+  - add_host: name={{ a_master }} groups=service_master
+
 - include: update.yml
 
+- name: Deploy OpenShift Services
+  hosts: service_master
+  connection: ssh
+  gather_facts: yes
+  roles:
+  - openshift_registry
+  - openshift_router
+
+- include: ../../common/openshift-cluster/create_services.yml
+  vars:
+     g_svc_master: "{{ service_master }}"
+
 - include: list.yml
diff --git a/playbooks/gce/openshift-cluster/list.yml b/playbooks/gce/openshift-cluster/list.yml
index 962381306..5ba0f5a48 100644
--- a/playbooks/gce/openshift-cluster/list.yml
+++ b/playbooks/gce/openshift-cluster/list.yml
@@ -16,7 +16,7 @@
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
     with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated)
 
-- name: List Hosts
+- name: List instance(s)
   hosts: oo_list_hosts
   gather_facts: no
   tasks:
diff --git a/roles/fluentd_master/tasks/main.yml b/roles/fluentd_master/tasks/main.yml
new file mode 100644
index 000000000..28caaa5b8
--- /dev/null
+++ b/roles/fluentd_master/tasks/main.yml
@@ -0,0 +1,46 @@
+---
+# TODO: Update fluentd install and configuration when packaging is complete
+- name: download and install td-agent
+  yum:
+    name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm'
+    state: present
+
+- name: Verify fluentd plugin installed
+  command: '/opt/td-agent/embedded/bin/gem query -i fluent-plugin-kubernetes'
+  register: _fluent_plugin_check
+  ignore_errors: yes
+
+- name: install Kubernetes fluentd plugin
+  command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes'
+  when: _fluent_plugin_check.rc == 1
+
+- name: Creates directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0755
+  with_items: ['/etc/td-agent/config.d']
+
+- name: Add include to td-agent configuration
+  lineinfile:
+    dest: '/etc/td-agent/td-agent.conf'
+    regexp: '^@include config.d'
+    line: '@include config.d/*.conf'
+    state: present
+
+- name: install Kubernetes fluentd configuration file
+  template:
+    src: kubernetes.conf.j2
+    dest: /etc/td-agent/config.d/kubernetes.conf
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: ensure td-agent is running
+  service:
+    name: 'td-agent'
+    state: started
+    enabled: yes
+
diff --git a/roles/fluentd_master/templates/kubernetes.conf.j2 b/roles/fluentd_master/templates/kubernetes.conf.j2
new file mode 100644
index 000000000..7b5c86062
--- /dev/null
+++ b/roles/fluentd_master/templates/kubernetes.conf.j2
@@ -0,0 +1,9 @@
+<match kubernetes.**>
+    type file
+    path /var/log/td-agent/containers.log
+    time_slice_format %Y%m%d
+    time_slice_wait 10m
+    time_format %Y%m%dT%H%M%S%z
+    compress gzip
+    utc
+</match>
diff --git a/roles/fluentd_node/tasks/main.yml b/roles/fluentd_node/tasks/main.yml
new file mode 100644
index 000000000..2526057cb
--- /dev/null
+++ b/roles/fluentd_node/tasks/main.yml
@@ -0,0 +1,54 @@
+---
+# TODO: Update fluentd install and configuration when packaging is complete
+- name: download and install td-agent
+  yum:
+    name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm'
+    state: present
+
+- name: Verify fluentd plugin installed
+  command: '/opt/td-agent/embedded/bin/gem query -i fluent-plugin-kubernetes'
+  register: _fluent_plugin_check
+  ignore_errors: yes
+
+- name: install Kubernetes fluentd plugin
+  command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes'
+  when: _fluent_plugin_check.rc == 1
+
+- name: Override td-agent configuration file
+  template:
+    src: td-agent.j2
+    dest: /etc/sysconfig/td-agent
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: Creates directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0755
+  with_items: ['/etc/td-agent/config.d', '/var/log/td-agent/tmp']
+
+- name: Add include to td-agent configuration
+  lineinfile:
+    dest: '/etc/td-agent/td-agent.conf'
+    regexp: '^@include config.d'
+    line: '@include config.d/*.conf'
+    state: present
+
+- name: install Kubernetes fluentd configuration file
+  template:
+    src: kubernetes.conf.j2
+    dest: /etc/td-agent/config.d/kubernetes.conf
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: ensure td-agent is running
+  service:
+    name: 'td-agent'
+    state: started
+    enabled: yes
+
diff --git a/roles/fluentd_node/templates/kubernetes.conf.j2 b/roles/fluentd_node/templates/kubernetes.conf.j2
new file mode 100644
index 000000000..5f1eecb20
--- /dev/null
+++ b/roles/fluentd_node/templates/kubernetes.conf.j2
@@ -0,0 +1,53 @@
+<source>
+  type tail
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/log/td-agent/tmp/fluentd-docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.*
+  format json
+  read_from_head true
+</source>
+
+<match docker.var.lib.docker.containers.*.*.log>
+  type kubernetes
+  container_id ${tag_parts[5]}
+  tag docker.${name}
+</match>
+
+<match kubernetes>
+  type copy
+
+  <store>
+    type forward
+    send_timeout 60s
+    recover_wait 10s
+    heartbeat_interval 1s
+    phi_threshold 16
+    hard_timeout 60s
+    log_level trace
+    require_ack_response true
+    heartbeat_type tcp
+
+    <server>
+      name {{groups['oo_first_master'][0]}}
+      host {{hostvars[groups['oo_first_master'][0]].openshift.common.hostname}}
+      port 24224
+      weight 60
+    </server>
+
+    <secondary>
+      type file
+      path /var/log/td-agent/forward-failed
+    </secondary>
+  </store>
+
+  <store>
+    type file
+    path /var/log/td-agent/containers.log
+    time_slice_format %Y%m%d
+    time_slice_wait 10m
+    time_format %Y%m%dT%H%M%S%z
+    compress gzip
+    utc
+  </store>
+</match>
diff --git a/roles/fluentd_node/templates/td-agent.j2 b/roles/fluentd_node/templates/td-agent.j2
new file mode 100644
index 000000000..7245e11ec
--- /dev/null
+++ b/roles/fluentd_node/templates/td-agent.j2
@@ -0,0 +1,2 @@
+DAEMON_ARGS=
+TD_AGENT_ARGS="/usr/sbin/td-agent --log /var/log/td-agent/td-agent.log --use-v1-config"
diff --git a/roles/openshift_common/tasks/main.yml b/roles/openshift_common/tasks/main.yml
index c55677c3f..5bd8690a7 100644
--- a/roles/openshift_common/tasks/main.yml
+++ b/roles/openshift_common/tasks/main.yml
@@ -10,6 +10,7 @@
       public_hostname: "{{ openshift_public_hostname | default(None) }}"
       public_ip: "{{ openshift_public_ip | default(None) }}"
       use_openshift_sdn: "{{ openshift_use_openshift_sdn | default(None) }}"
+      use_fluentd: "{{ openshift_use_fluentd | default(True) }}"
       deployment_type: "{{ openshift_deployment_type }}"
 - name: Set hostname
   hostname: name={{ openshift.common.hostname }}
diff --git a/roles/openshift_master/README.md b/roles/openshift_master/README.md
index 9f9d0a613..3178e318c 100644
--- a/roles/openshift_master/README.md
+++ b/roles/openshift_master/README.md
@@ -17,7 +17,7 @@ From this role:
 |-------------------------------------|-----------------------|--------------------------------------------------|
 | openshift_master_debug_level        | openshift_debug_level | Verbosity of the debug logs for openshift-master |
 | openshift_node_ips                  | []                    | List of the openshift node ip addresses to pre-register when openshift-master starts up |
-| openshift_registry_url              | UNDEF                 | Default docker registry to use |
+| oreg_url                            | UNDEF                 | Default docker registry to use |
 | openshift_master_api_port           | UNDEF                 | |
 | openshift_master_console_port       | UNDEF                 | |
 | openshift_master_api_url            | UNDEF                 | |
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 56cf43531..11195e83e 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -11,6 +11,10 @@ os_firewall_allow:
   port: 53/tcp
 - service: OpenShift dns udp
   port: 53/udp
+- service: Fluentd td-agent tcp
+  port: 24224/tcp
+- service: Fluentd td-agent udp
+  port: 24224/udp
 os_firewall_deny:
 - service: OpenShift api http
   port: 8080/tcp
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index f9e6199a5..ac96e2b48 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -49,15 +49,15 @@
 # TODO: should probably use a template lookup for this
 # TODO: should allow for setting --etcd, --kubernetes options
 # TODO: recreate config if values change
-- name: Use enterprise default for openshift_registry_url if not set
+- name: Use enterprise default for oreg_url if not set
   set_fact:
-    openshift_registry_url: "openshift3_beta/ose-${component}:${version}"
-  when: openshift.common.deployment_type == 'enterprise' and openshift_registry_url is not defined
+    oreg_url: "openshift3_beta/ose-${component}:${version}"
+  when: openshift.common.deployment_type == 'enterprise' and oreg_url is not defined
 
-- name: Use online default for openshift_registry_url if not set
+- name: Use online default for oreg_url if not set
   set_fact:
-    openshift_registry_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}"
-  when: openshift.common.deployment_type == 'online' and openshift_registry_url is not defined
+    oreg_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}"
+  when: openshift.common.deployment_type == 'online' and oreg_url is not defined
 
 - name: Create master config
   command: >
@@ -67,7 +67,7 @@
     --master={{ openshift.master.api_url }}
     --public-master={{ openshift.master.public_api_url }}
     --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://0.0.0.0:{{ openshift.master.api_port }}
-    {{ ('--images=' ~ openshift_registry_url) if (openshift_registry_url | default('', true) != '') else '' }}
+    {{ ('--images=' ~ oreg_url) if (oreg_url | default('', true) != '') else '' }}
     {{ ('--nodes=' ~ openshift_node_ips | join(',')) if (openshift_node_ips | default('', true) != '') else '' }}
   args:
     chdir: "{{ openshift_cert_parent_dir }}"
diff --git a/roles/openshift_node/README.md b/roles/openshift_node/README.md
index 83359f164..c3c17b848 100644
--- a/roles/openshift_node/README.md
+++ b/roles/openshift_node/README.md
@@ -17,7 +17,7 @@ From this role:
 | Name                                     | Default value         |                                        |
 |------------------------------------------|-----------------------|----------------------------------------|
 | openshift_node_debug_level               | openshift_debug_level | Verbosity of the debug logs for openshift-node |
-| openshift_registry_url                   | UNDEF (Optional)      | Default docker registry to use |
+| oreg_url                                 | UNDEF (Optional)      | Default docker registry to use |
 
 From openshift_common:
 | Name                          |  Default Value      |                     | 
diff --git a/roles/openshift_register_nodes/tasks/main.yml b/roles/openshift_register_nodes/tasks/main.yml
index d4d72d126..dcb96bbf9 100644
--- a/roles/openshift_register_nodes/tasks/main.yml
+++ b/roles/openshift_register_nodes/tasks/main.yml
@@ -6,15 +6,15 @@
 
 # TODO: use a template lookup here
 # TODO: create a failed_when condition
-- name: Use enterprise default for openshift_registry_url if not set
+- name: Use enterprise default for oreg_url if not set
   set_fact:
-    openshift_registry_url: "openshift3_beta/ose-${component}:${version}"
-  when: openshift.common.deployment_type == 'enterprise' and openshift_registry_url is not defined
+    oreg_url: "openshift3_beta/ose-${component}:${version}"
+  when: openshift.common.deployment_type == 'enterprise' and oreg_url is not defined
 
-- name: Use online default for openshift_registry_url if not set
+- name: Use online default for oreg_url if not set
   set_fact:
-    openshift_registry_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}"
-  when: openshift.common.deployment_type == 'online' and openshift_registry_url is not defined
+    oreg_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}"
+  when: openshift.common.deployment_type == 'online' and oreg_url is not defined
 
 - name: Create node config
   command: >
@@ -30,7 +30,7 @@
       --certificate-authority={{ openshift_master_ca_cert }}
       --signer-serial={{ openshift_master_ca_dir }}/serial.txt
       --node-client-certificate-authority={{ openshift_master_ca_cert }}
-      {{ ('--images=' ~ openshift_registry_url) if openshift_registry_url is defined else '' }}
+      {{ ('--images=' ~ oreg_url) if oreg_url is defined else '' }}
       --listen=https://0.0.0.0:10250
   args:
     chdir: "{{ openshift_cert_parent_dir }}"
diff --git a/roles/openshift_registry/README.md b/roles/openshift_registry/README.md
new file mode 100644
index 000000000..202c818b8
--- /dev/null
+++ b/roles/openshift_registry/README.md
@@ -0,0 +1,42 @@
+OpenShift Container Docker Registry
+===================================
+
+OpenShift Docker Registry  service installation
+
+Requirements
+------------
+
+Running OpenShift cluster
+
+Role Variables
+--------------
+
+From this role:
+| Name               | Default value                                         |                     |
+|--------------------|-------------------------------------------------------|---------------------|
+|                    |                                                       |                     |
+
+From openshift_common:
+| Name                  | Default value |                                      |
+|-----------------------|---------------|--------------------------------------|
+| openshift_debug_level | 0             | Global openshift debug log verbosity |
+
+
+Dependencies
+------------
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Red Hat openshift@redhat.com
+
diff --git a/roles/openshift_registry/handlers/main.yml b/roles/openshift_registry/handlers/main.yml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/roles/openshift_registry/handlers/main.yml
diff --git a/roles/openshift_registry/meta/main.yml b/roles/openshift_registry/meta/main.yml
new file mode 100644
index 000000000..93b6797d1
--- /dev/null
+++ b/roles/openshift_registry/meta/main.yml
@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: OpenShift Red Hat
+  description: OpenShift Embedded Docker Registry
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.7
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
diff --git a/roles/openshift_registry/tasks/main.yml b/roles/openshift_registry/tasks/main.yml
new file mode 100644
index 000000000..7e6982d99
--- /dev/null
+++ b/roles/openshift_registry/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- set_fact: _oreg_images="--images={{ oreg_url|quote }}"
+  when: oreg_url is defined
+
+- name: Deploy OpenShift Registry
+  command: openshift admin registry --create --credentials=/var/lib/openshift/openshift.local.certificates/openshift-registry/.kubeconfig {{ _oreg_images|default() }}
+  register: _oreg_results
+  changed_when: "'service exists' not in _oreg_results.stdout"
diff --git a/roles/openshift_registry/vars/main.yml b/roles/openshift_registry/vars/main.yml
new file mode 100644
index 000000000..cd21505a4
--- /dev/null
+++ b/roles/openshift_registry/vars/main.yml
@@ -0,0 +1,2 @@
+---
+
diff --git a/roles/openshift_router/README.md b/roles/openshift_router/README.md
new file mode 100644
index 000000000..6d8ee25c6
--- /dev/null
+++ b/roles/openshift_router/README.md
@@ -0,0 +1,41 @@
+OpenShift Container Router
+==========================
+
+OpenShift Router service installation
+
+Requirements
+------------
+
+Running OpenShift cluster
+
+Role Variables
+--------------
+
+From this role:
+| Name               | Default value                                         |                     |
+|--------------------|-------------------------------------------------------|---------------------|
+|                    |                                                       |                     |
+
+From openshift_common:
+| Name                  | Default value |                                      |
+|-----------------------|---------------|--------------------------------------|
+| openshift_debug_level | 0             | Global openshift debug log verbosity |
+
+Dependencies
+------------
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License, Version 2.0
+
+Author Information
+------------------
+
+Red Hat openshift@redhat.com
+
diff --git a/roles/openshift_router/handlers/main.yml b/roles/openshift_router/handlers/main.yml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/roles/openshift_router/handlers/main.yml
diff --git a/roles/openshift_router/meta/main.yml b/roles/openshift_router/meta/main.yml
new file mode 100644
index 000000000..0471e5e14
--- /dev/null
+++ b/roles/openshift_router/meta/main.yml
@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: OpenShift Red Hat
+  description: OpenShift Embedded Router
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.7
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
diff --git a/roles/openshift_router/tasks/main.yml b/roles/openshift_router/tasks/main.yml
new file mode 100644
index 000000000..f1ee99dd3
--- /dev/null
+++ b/roles/openshift_router/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- set_fact: _ortr_images="--images={{ oreg_url|quote }}"
+  when: oreg_url is defined
+
+- name: Deploy OpenShift Router
+  command: openshift ex router --create --credentials=/var/lib/openshift/openshift.local.certificates/openshift-router/.kubeconfig {{ _ortr_images|default() }}
+  register: _ortr_results
+  changed_when: "'service exists' not in _ortr_results.stdout"
diff --git a/roles/openshift_router/vars/main.yml b/roles/openshift_router/vars/main.yml
new file mode 100644
index 000000000..cd21505a4
--- /dev/null
+++ b/roles/openshift_router/vars/main.yml
@@ -0,0 +1,2 @@
+---
+