diff options
27 files changed, 603 insertions, 83 deletions
diff --git a/.tito/packages/openshift-ansible b/.tito/packages/openshift-ansible index 1ca23082d..73f2e8c87 100644 --- a/.tito/packages/openshift-ansible +++ b/.tito/packages/openshift-ansible @@ -1 +1 @@ -3.9.0-0.10.0 ./ +3.9.0-0.13.0 ./ diff --git a/openshift-ansible.spec b/openshift-ansible.spec index 2be8ebd5b..b65c8d252 100644 --- a/openshift-ansible.spec +++ b/openshift-ansible.spec @@ -10,7 +10,7 @@ Name: openshift-ansible Version: 3.9.0 -Release: 0.10.0%{?dist} +Release: 0.13.0%{?dist} Summary: Openshift and Atomic Enterprise Ansible License: ASL 2.0 URL: https://github.com/openshift/openshift-ansible @@ -202,6 +202,54 @@ Atomic OpenShift Utilities includes %changelog +* Tue Jan 02 2018 Jenkins CD Merge Bot <smunilla@redhat.com> 3.9.0-0.13.0 +- Bug 1527178 - installation of logging stack failed: Invalid version specified + for Elasticsearch (nhosoi@redhat.com) +- Remove bootstrap.yml from main.yml in openshift_node role + (mgugino@redhat.com) + +* Tue Jan 02 2018 Jenkins CD Merge Bot <smunilla@redhat.com> 3.9.0-0.12.0 +- + +* Mon Jan 01 2018 Jenkins CD Merge Bot <smunilla@redhat.com> 3.9.0-0.11.0 +- aws: Fix misnamed variable in provisioning_vars.yml.example + (mbarnes@fedoraproject.org) +- Fix container_runtime openshift_containerized_host_groups + (mgugino@redhat.com) +- Remove references to deployment_type (mgugino@redhat.com) +- Must directly specify google-cloud-sdk version (ccoleman@redhat.com) +- daemonset config role. (kwoodson@redhat.com) +- Move validate_hosts to prerequisites.yml (mgugino@redhat.com) +- Move sanity_checks into custom action plugin (mgugino@redhat.com) +- Remove openshift.common.{is_atomic|is_containerized} (mgugino@redhat.com) +- Adding support for docker-storage-setup on overlay (kwoodson@redhat.com) +- Add gcloud to the installer image (ccoleman@redhat.com) +- Remove some small items from openshift_facts (mgugino@redhat.com) +- Relocate filter plugins to lib_utils (mgugino@redhat.com) +- Fix hosted_reg_router selectors (mgugino@redhat.com) +- set repos after registration: convert to match task -> import_role model. + (markllama@gmail.com) +- Remove openshift_node_facts role (mgugino@redhat.com) +- Move node group tags to openshift_aws_{master,node}_group. + (abutcher@redhat.com) +- Add CentOS-OpenShift-Origin37 repo template. (abutcher@redhat.com) +- Adding no_log to registry_auth. (kwoodson@redhat.com) +- Fix rhel_repos disable command (mazzystr@gmail.com) +- Fix rhel_subscribe boolean (mgugino@redhat.com) +- Move repo and subscribe to prerequisites (mgugino@redhat.com) +- Deprecate using Ansible tests as filters (rteague@redhat.com) +- Removing config trigger for ES DC, updating to use a handler to rollout ES at + the end of a deployment, allowing for override with variable + (ewolinet@redhat.com) +- openshift_logging_{fluentd,mux}_file_buffer_limit mismatch + (nhosoi@redhat.com) +- Update version check to Ansible 2.4.1 (rteague@redhat.com) +- Remove openshift_node_facts part 1 (mgugino@redhat.com) +- Validate node hostname and IP address (rteague@redhat.com) +- Add missing openshift_service_type (mgugino@redhat.com) +- prevent TSB pods from spinning on inappropriate nodes (jminter@redhat.com) +- Add readiness probe to kuryr controller pod (ltomasbo@redhat.com) + * Thu Dec 14 2017 Jenkins CD Merge Bot <smunilla@redhat.com> 3.9.0-0.10.0 - Bump requirements.txt to Ansible 2.4.1 (rteague@redhat.com) - Commit to stabalize RHSM operations. This code is derived from contrib diff --git a/playbooks/openshift-node/private/image_prep.yml b/playbooks/openshift-node/private/image_prep.yml index 6b517197d..c0ddcd926 100644 --- a/playbooks/openshift-node/private/image_prep.yml +++ b/playbooks/openshift-node/private/image_prep.yml @@ -12,6 +12,13 @@ - name: run node config import_playbook: configure_nodes.yml +- name: node bootstrap config + hosts: oo_nodes_to_config:!oo_containerized_master_nodes + tasks: + - include_role: + name: openshift_node + tasks_from: bootstrap.yml + - name: Re-enable excluders import_playbook: enable_excluders.yml diff --git a/roles/container_runtime/tasks/registry_auth.yml b/roles/container_runtime/tasks/registry_auth.yml index 2c7bc5711..4f1abd59a 100644 --- a/roles/container_runtime/tasks/registry_auth.yml +++ b/roles/container_runtime/tasks/registry_auth.yml @@ -15,6 +15,7 @@ - not openshift_docker_alternative_creds | bool - oreg_auth_user is defined - (not docker_cli_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool + no_log: True # docker_creds is a custom module from lib_utils # 'docker login' requires a docker.service running on the local host, this is an @@ -30,3 +31,4 @@ - openshift_docker_alternative_creds | bool - oreg_auth_user is defined - (not docker_cli_auth_credentials_stat.stat.exists or oreg_auth_credentials_replace) | bool + no_log: True diff --git a/roles/openshift_sanitize_inventory/library/conditional_set_fact.py b/roles/lib_openshift/library/conditional_set_fact.py index f61801714..363399f33 100644 --- a/roles/openshift_sanitize_inventory/library/conditional_set_fact.py +++ b/roles/lib_openshift/library/conditional_set_fact.py @@ -29,6 +29,10 @@ EXAMPLES = ''' fact1: not_defined_variable fact2: defined_variable +- name: Conditionally set fact falling back on default + conditional_set_fact: + fact1: not_defined_var | defined_variable + ''' @@ -48,12 +52,14 @@ def run_module(): is_changed = False for param in module.params['vars']: - other_var = module.params['vars'][param] - - if other_var in module.params['facts']: - local_facts[param] = module.params['facts'][other_var] - if not is_changed: - is_changed = True + other_vars = module.params['vars'][param].replace(" ", "") + + for other_var in other_vars.split('|'): + if other_var in module.params['facts']: + local_facts[param] = module.params['facts'][other_var] + if not is_changed: + is_changed = True + break return module.exit_json(changed=is_changed, # noqa: F405 ansible_facts=local_facts) diff --git a/roles/openshift_daemonset_config/defaults/main.yml b/roles/openshift_daemonset_config/defaults/main.yml new file mode 100644 index 000000000..ebe5671d2 --- /dev/null +++ b/roles/openshift_daemonset_config/defaults/main.yml @@ -0,0 +1,19 @@ +--- +openshift_daemonset_config_namespace: openshift-node +openshift_daemonset_config_daemonset_name: ops-node-config +openshift_daemonset_config_configmap_name: "{{ openshift_daemonset_config_daemonset_name }}" +openshift_daemonset_config_node_selector: + config: config +openshift_daemonset_config_sa_name: ops +openshift_daemonset_config_configmap_files: {} +openshift_daemonset_config_configmap_literals: {} +openshift_daemonset_config_monitoring: False +openshift_daemonset_config_interval: 300 +openshift_daemonset_config_script: config.sh +openshift_daemonset_config_secret_name: operations-config-secret +openshift_daemonset_config_secrets: {} +openshift_daemonset_config_runasuser: 0 +openshift_daemonset_config_privileged: True +openshift_daemonset_config_resources: + cpu: 10m + memory: 10Mi diff --git a/roles/openshift_daemonset_config/meta/main.yml b/roles/openshift_daemonset_config/meta/main.yml new file mode 100644 index 000000000..d2bbd2576 --- /dev/null +++ b/roles/openshift_daemonset_config/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- lib_openshift diff --git a/roles/openshift_daemonset_config/tasks/main.yml b/roles/openshift_daemonset_config/tasks/main.yml new file mode 100644 index 000000000..450cc9dca --- /dev/null +++ b/roles/openshift_daemonset_config/tasks/main.yml @@ -0,0 +1,58 @@ +--- +- name: add a sa + oc_serviceaccount: + name: "{{ openshift_daemonset_config_sa_name }}" + namespace: "{{ openshift_daemonset_config_namespace }}" + +- name: add sa to privileged scc + oc_adm_policy_user: + namespace: "{{ openshift_daemonset_config_namespace }}" + resource_kind: scc + resource_name: privileged + state: present + user: "system:serviceaccount:{{ openshift_daemonset_config_namespace }}:{{ openshift_daemonset_config_sa_name }}" + +- name: copy template to disk + template: + dest: "/tmp/{{ item.name }}" + src: "{{ item.name }}.j2" + with_items: + - name: daemonset.yml + +- name: copy files to disk + copy: + src: "{{ item.key }}" + dest: "{{ item.value }}" + with_dict: "{{ openshift_daemonset_config_configmap_files }}" + +- name: create the namespace + oc_project: + state: present + name: "{{ openshift_daemonset_config_namespace }}" + +- name: lay down secrets + oc_secret: + state: present + name: "{{ openshift_daemonset_config_secret_name }}" + namespace: "{{ openshift_daemonset_config_namespace }}" + delete_after: true + contents: "{{ openshift_daemonset_config_secrets }}" + when: + - openshift_daemonset_config_secrets != {} + +- name: create the configmap + oc_configmap: + state: present + name: "{{ openshift_daemonset_config_configmap_name }}" + namespace: "{{ openshift_daemonset_config_namespace }}" + from_literal: "{{ openshift_daemonset_config_configmap_literals }}" + from_file: "{{ openshift_daemonset_config_configmap_files }}" + +- name: deploy daemonset + oc_obj: + state: present + namespace: "{{ openshift_daemonset_config_namespace }}" # openshift-node?? + name: "{{ openshift_daemonset_config_daemonset_name }}" + kind: daemonset + files: + - /tmp/daemonset.yml diff --git a/roles/openshift_daemonset_config/templates/daemonset.yml.j2 b/roles/openshift_daemonset_config/templates/daemonset.yml.j2 new file mode 100644 index 000000000..9792f6d16 --- /dev/null +++ b/roles/openshift_daemonset_config/templates/daemonset.yml.j2 @@ -0,0 +1,142 @@ +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: {{ openshift_daemonset_config_daemonset_name }} + annotations: + kubernetes.io/description: | + This daemon set manages the operational configuration for a cluster and ensures all nodes have + a concrete set of config in place. It could also use a local ansible run against the /host directory. +spec: + selector: + matchLabels: + app: {{ openshift_daemonset_config_daemonset_name }} + confighosts: ops + ops.openshift.io/role: operations + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ openshift_daemonset_config_daemonset_name }} + confighosts: ops + ops.openshift.io/role: operations + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: +{% if openshift_daemonset_config_node_selector is defined and openshift_daemonset_config_node_selector != {} %} + nodeSelector: {{ openshift_daemonset_config_node_selector | to_json }} +{% endif %} + serviceAccountName: {{ openshift_daemonset_config_sa_name }} + hostNetwork: true + hostPID: true + hostIPC: true + containers: + - name: config + image: centos:7 + env: + - name: RESYNC_INTERVAL + value: "{{ openshift_daemonset_config_interval }}" + command: + - /bin/bash + - -c + - | + #!/bin/sh + set -o errexit + + while true; do + + # execute user defined script + sh /opt/config/{{ openshift_daemonset_config_script }} + + # sleep for ${RESYNC_INTERVAL} minutes, then loop. if we fail Kubelet will restart us again + echo "Success, sleeping for ${RESYNC_INTERVAL}s" + exec sleep ${RESYNC_INTERVAL} + + # Return to perform the config + done + securityContext: + # Must be root to modify host system + runAsUser: {{ openshift_daemonset_config_runasuser }} + # Permission could be reduced by selecting an appropriate SELinux policy that allows + # us to update the named directories + privileged: {{ openshift_daemonset_config_privileged }} + volumeMounts: + # Directory which contains the host volume. + - mountPath: /host + name: host + # Our node configuration + - mountPath: /opt/config + name: config +{% if openshift_daemonset_config_secrets != {} %} + # Our delivered secrets + - mountPath: /opt/secrets + name: secrets +{% endif %} + resources: + requests: + cpu: {{ openshift_daemonset_config_resources.cpu }} + memory: {{ openshift_daemonset_config_resources.memory }} +{% if openshift_daemonset_config_monitoring %} + - name: monitoring + image: openshifttools/oso-centos7-host-monitoring:latest + securityContext: + # Must be root to read content + runAsUser: 0 + privileged: true + + volumeMounts: + - mountPath: /host + name: host + readOnly: true + - mountPath: /etc/localtime + subPath: etc/localtime + name: host + readOnly: true + - mountPath: /sys + subPath: sys + name: host + readOnly: true + - mountPath: /var/run/docker.sock + subPath: var/run/docker.sock + name: host + readOnly: true + - mountPath: /var/run/openvswitch + subPath: var/run/openvswitch + name: host + readOnly: true + - mountPath: /etc/origin + subPath: etc/origin + name: host + readOnly: true + - mountPath: /usr/bin/oc + subPath: usr/bin/oc + name: host + readOnly: true + name: host + readOnly: true + - mountPath: /host/var/cache/yum + subPath: var/cache/yum + name: host + - mountPath: /container_setup/monitoring-config.yml + subPath: monitoring-config.yaml + name: config + - mountPath: /opt/config + name: config + resources: + requests: + cpu: 10m + memory: 10Mi +{% endif %} + volumes: + - name: config + configMap: + name: {{ openshift_daemonset_config_configmap_name }} +{% if openshift_daemonset_config_secrets != {} %} + - name: secrets + secret: + secretName: {{ openshift_daemonset_config_secret_name }} +{% endif %} + - name: host + hostPath: + path: / diff --git a/roles/openshift_logging/filter_plugins/openshift_logging.py b/roles/openshift_logging/filter_plugins/openshift_logging.py index e1a5ea726..ba412b5a6 100644 --- a/roles/openshift_logging/filter_plugins/openshift_logging.py +++ b/roles/openshift_logging/filter_plugins/openshift_logging.py @@ -102,6 +102,28 @@ def serviceaccount_namespace(qualified_sa, default=None): return seg[-1] +def flatten_dict(data, parent_key=None): + """ This filter plugin will flatten a dict and its sublists into a single dict + """ + if not isinstance(data, dict): + raise RuntimeError("flatten_dict failed, expects to flatten a dict") + + merged = dict() + + for key in data: + if parent_key is not None: + insert_key = '.'.join((parent_key, key)) + else: + insert_key = key + + if isinstance(data[key], dict): + merged.update(flatten_dict(data[key], insert_key)) + else: + merged[insert_key] = data[key] + + return merged + + # pylint: disable=too-few-public-methods class FilterModule(object): ''' OpenShift Logging Filters ''' @@ -117,5 +139,6 @@ class FilterModule(object): 'es_storage': es_storage, 'serviceaccount_name': serviceaccount_name, 'serviceaccount_namespace': serviceaccount_namespace, - 'walk': walk + 'walk': walk, + "flatten_dict": flatten_dict } diff --git a/roles/openshift_logging/library/logging_patch.py b/roles/openshift_logging/library/logging_patch.py new file mode 100644 index 000000000..d2c0bc456 --- /dev/null +++ b/roles/openshift_logging/library/logging_patch.py @@ -0,0 +1,112 @@ +#!/usr/bin/python + +""" Ansible module to help with creating context patch file with whitelisting for logging """ + +import difflib +import re + +from ansible.module_utils.basic import AnsibleModule + + +DOCUMENTATION = ''' +--- +module: logging_patch + +short_description: This will create a context patch file while giving ability + to whitelist some lines (excluding them from comparison) + +description: + - "To create configmap patches for logging" + +author: + - Eric Wolinetz ewolinet@redhat.com +''' + + +EXAMPLES = ''' +- logging_patch: + original_file: "{{ tempdir }}/current.yml" + new_file: "{{ configmap_new_file }}" + whitelist: "{{ configmap_protected_lines | default([]) }}" + +''' + + +def account_for_whitelist(file_contents, white_list=None): + """ This method will remove lines that contain whitelist values from the content + of the file so that we aren't build a patch based on that line + + Usage: + + for file_contents: + + index: + number_of_shards: {{ es_number_of_shards | default ('1') }} + number_of_replicas: {{ es_number_of_replicas | default ('0') }} + unassigned.node_left.delayed_timeout: 2m + translog: + flush_threshold_size: 256mb + flush_threshold_period: 5m + + + and white_list: + + ['number_of_shards', 'number_of_replicas'] + + + We would end up with: + + index: + unassigned.node_left.delayed_timeout: 2m + translog: + flush_threshold_size: 256mb + flush_threshold_period: 5m + + """ + + for line in white_list: + file_contents = re.sub(r".*%s:.*\n" % line, "", file_contents) + + return file_contents + + +def run_module(): + """ The body of the module, we check if the variable name specified as the value + for the key is defined. If it is then we use that value as for the original key """ + + module = AnsibleModule( + argument_spec=dict( + original_file=dict(type='str', required=True), + new_file=dict(type='str', required=True), + whitelist=dict(required=False, type='list', default=[]) + ), + supports_check_mode=True + ) + + original_fh = open(module.params['original_file'], "r") + original_contents = original_fh.read() + original_fh.close() + + original_contents = account_for_whitelist(original_contents, module.params['whitelist']) + + new_fh = open(module.params['new_file'], "r") + new_contents = new_fh.read() + new_fh.close() + + new_contents = account_for_whitelist(new_contents, module.params['whitelist']) + + uni_diff = difflib.unified_diff(new_contents.splitlines(), + original_contents.splitlines(), + lineterm='') + + return module.exit_json(changed=False, # noqa: F405 + raw_patch="\n".join(uni_diff)) + + +def main(): + """ main """ + run_module() + + +if __name__ == '__main__': + main() diff --git a/roles/openshift_logging/library/openshift_logging_facts.py b/roles/openshift_logging/library/openshift_logging_facts.py index 98d0d1c4f..302a9b4c9 100644 --- a/roles/openshift_logging/library/openshift_logging_facts.py +++ b/roles/openshift_logging/library/openshift_logging_facts.py @@ -204,6 +204,14 @@ class OpenshiftLoggingFacts(OCBaseCommand): if comp is not None: self.add_facts_for(comp, "services", name, dict()) + # pylint: disable=too-many-arguments + def facts_from_configmap(self, comp, kind, name, config_key, yaml_file=None): + '''Extracts facts in logging namespace from configmap''' + if yaml_file is not None: + config_facts = yaml.load(yaml_file) + self.facts[comp][kind][name][config_key] = config_facts + self.facts[comp][kind][name]["raw"] = yaml_file + def facts_for_configmaps(self, namespace): ''' Gathers facts for configmaps in logging namespace ''' self.default_keys_for("configmaps") @@ -214,7 +222,10 @@ class OpenshiftLoggingFacts(OCBaseCommand): name = item["metadata"]["name"] comp = self.comp(name) if comp is not None: - self.add_facts_for(comp, "configmaps", name, item["data"]) + self.add_facts_for(comp, "configmaps", name, dict(item["data"])) + if comp in ["elasticsearch", "elasticsearch_ops"]: + for config_key in item["data"]: + self.facts_from_configmap(comp, "configmaps", name, config_key, item["data"][config_key]) def facts_for_oauthclients(self, namespace): ''' Gathers facts for oauthclients used with logging ''' diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml index 11f59652c..913478027 100644 --- a/roles/openshift_logging/tasks/install_logging.yaml +++ b/roles/openshift_logging/tasks/install_logging.yaml @@ -4,6 +4,9 @@ oc_bin: "{{openshift_client_binary}}" openshift_logging_namespace: "{{openshift_logging_namespace}}" +## This is include vs import because we need access to group/inventory variables +- include_tasks: set_defaults_from_current.yml + - name: Set logging project oc_project: state: present diff --git a/roles/openshift_logging/tasks/patch_configmap_file.yaml b/roles/openshift_logging/tasks/patch_configmap_file.yaml new file mode 100644 index 000000000..30087fe6a --- /dev/null +++ b/roles/openshift_logging/tasks/patch_configmap_file.yaml @@ -0,0 +1,35 @@ +--- +## The purpose of this task file is to get a patch that is based on the diff +## between configmap_current_file and configmap_new_file. The module +## logging_patch takes the paths of two files to compare and also a list of +## variables whose line we exclude from the diffs. +## We then patch the new configmap file so that we can build a configmap +## using that file later. We then use oc apply to idempotenly modify any +## existing configmap. + +## The following variables are expected to be provided when including this task: +# __configmap_output -- This is provided to us from patch_configmap_files.yaml +# it is a dict of the configmap where configmap_current_file exists +# configmap_current_file -- The name of the data file in the __configmap_output +# configmap_new_file -- The path to the file that we intend to oc apply later +# we apply our generated patch to this file. +# configmap_protected_lines -- The list of variables to exclude from the diff + +- copy: + content: "{{ __configmap_output.results.results[0]['data'][configmap_current_file] }}" + dest: "{{ tempdir }}/current.yml" + +- logging_patch: + original_file: "{{ tempdir }}/current.yml" + new_file: "{{ configmap_new_file }}" + whitelist: "{{ configmap_protected_lines | default([]) }}" + register: patch_output + +- copy: + content: "{{ patch_output.raw_patch }}\n" + dest: "{{ tempdir }}/patch.patch" + when: patch_output.raw_patch | length > 0 + +- command: > + patch --force --quiet -u "{{ configmap_new_file }}" "{{ tempdir }}/patch.patch" + when: patch_output.raw_patch | length > 0 diff --git a/roles/openshift_logging/tasks/patch_configmap_files.yaml b/roles/openshift_logging/tasks/patch_configmap_files.yaml new file mode 100644 index 000000000..74a9cc287 --- /dev/null +++ b/roles/openshift_logging/tasks/patch_configmap_files.yaml @@ -0,0 +1,31 @@ +--- +## The purpose of this task file is to take in a list of configmap files provided +## in the variable configmap_file_names, which correspond to the data sections +## within a configmap. We iterate over each of these files and create a patch +## from the diff between current_file and new_file to try to maintain any custom +## changes that a user may have made to a currently deployed configmap while +## trying to idempotently update with any role provided files. + +## The following variables are expected to be provided when including this task: +# configmap_name -- This is the name of the configmap that the files exist in +# configmap_namespace -- The namespace that the configmap lives in +# configmap_file_names -- This is expected to be passed in as a dict +# current_file -- The name of the data entry within the configmap +# new_file -- The file path to the file we are comparing to current_file +# protected_lines -- List of variables whose line will be excluded when creating a diff + +- oc_configmap: + name: "{{ configmap_name }}" + state: list + namespace: "{{ configmap_namespace }}" + register: __configmap_output + +- when: __configmap_output.results.stderr is undefined + include_tasks: patch_configmap_file.yaml + vars: + configmap_current_file: "{{ configmap_files.current_file }}" + configmap_new_file: "{{ configmap_files.new_file }}" + configmap_protected_lines: "{{ configmap_files.protected_lines | default([]) }}" + with_items: "{{ configmap_file_names }}" + loop_control: + loop_var: configmap_files diff --git a/roles/openshift_logging/tasks/set_defaults_from_current.yml b/roles/openshift_logging/tasks/set_defaults_from_current.yml new file mode 100644 index 000000000..dde362abe --- /dev/null +++ b/roles/openshift_logging/tasks/set_defaults_from_current.yml @@ -0,0 +1,34 @@ +--- + +## We are pulling default values from configmaps if they exist already +## Using conditional_set_fact allows us to set the value of a variable based on +## the value of another one, if it is already defined. Else we don't set the +## left hand side (it stays undefined as well). + +## conditional_set_fact allows us to specify a fact source, so first we try to +## set variables in the logging-elasticsearch & logging-elasticsearch-ops configmaps +## afterwards we set the value of the variable based on the value in the inventory +## but fall back to using the value from a configmap as a default. If neither is set +## then the variable remains undefined and the role default will be used. + +- conditional_set_fact: + facts: "{{ openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch']['elasticsearch.yml'] | flatten_dict }}" + vars: + __openshift_logging_es_number_of_shards: index.number_of_shards + __openshift_logging_es_number_of_replicas: index.number_of_replicas + when: openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch'] is defined + +- conditional_set_fact: + facts: "{{ openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops']['elasticsearch.yml'] | flatten_dict }}" + vars: + __openshift_logging_es_ops_number_of_shards: index.number_of_shards + __openshift_logging_es_ops_number_of_replicas: index.number_of_replicas + when: openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops'] is defined + +- conditional_set_fact: + facts: "{{ hostvars[inventory_hostname] }}" + vars: + openshift_logging_es_number_of_shards: openshift_logging_es_number_of_shards | __openshift_logging_es_number_of_shards + openshift_logging_es_number_of_replicas: openshift_logging_es_number_of_replicas | __openshift_logging_es_number_of_replicas + openshift_logging_es_ops_number_of_shards: openshift_logging_es_ops_number_of_shards | __openshift_logging_es_ops_number_of_shards + openshift_logging_es_ops_number_of_replicas: openshift_logging_es_ops_number_of_replicas | __openshift_logging_es_ops_number_of_replicas diff --git a/roles/openshift_logging_curator/tasks/main.yaml b/roles/openshift_logging_curator/tasks/main.yaml index 524e239b7..53b464113 100644 --- a/roles/openshift_logging_curator/tasks/main.yaml +++ b/roles/openshift_logging_curator/tasks/main.yaml @@ -54,14 +54,17 @@ - copy: src: curator.yml dest: "{{ tempdir }}/curator.yml" - when: curator_config_contents is undefined changed_when: no -- copy: - content: "{{ curator_config_contents }}" - dest: "{{ tempdir }}/curator.yml" - when: curator_config_contents is defined - changed_when: no +- include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-curator" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "config.yaml" + new_file: "{{ tempdir }}/curator.yml" - name: Set Curator configmap oc_configmap: diff --git a/roles/openshift_logging_curator/vars/main.yml b/roles/openshift_logging_curator/vars/main.yml index 95bf462d1..5bee58725 100644 --- a/roles/openshift_logging_curator/vars/main.yml +++ b/roles/openshift_logging_curator/vars/main.yml @@ -1,3 +1,3 @@ --- -__latest_curator_version: "3_6" -__allowed_curator_versions: ["3_5", "3_6", "3_7"] +__latest_curator_version: "3_8" +__allowed_curator_versions: ["3_5", "3_6", "3_7", "3_8"] diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml index 6ddeb122e..9e7646379 100644 --- a/roles/openshift_logging_elasticsearch/tasks/main.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml @@ -168,33 +168,31 @@ when: es_logging_contents is undefined changed_when: no -- set_fact: - __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}" - __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}" - - template: src: elasticsearch.yml.j2 dest: "{{ tempdir }}/elasticsearch.yml" vars: allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}" - es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}" - es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}" + es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}" + es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}" es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}" when: es_config_contents is undefined changed_when: no -- copy: - content: "{{ es_logging_contents }}" - dest: "{{ tempdir }}/elasticsearch-logging.yml" - when: es_logging_contents is defined - changed_when: no - -- copy: - content: "{{ es_config_contents }}" - dest: "{{ tempdir }}/elasticsearch.yml" - when: es_config_contents is defined - changed_when: no +# create diff between current configmap files and our current files +- include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-elasticsearch" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "elasticsearch.yml" + new_file: "{{ tempdir }}/elasticsearch.yml" + protected_lines: ["number_of_shards", "number_of_replicas"] + - current_file: "logging.yml" + new_file: "{{ tempdir }}/elasticsearch-logging.yml" - name: Set ES configmap oc_configmap: diff --git a/roles/openshift_logging_elasticsearch/vars/main.yml b/roles/openshift_logging_elasticsearch/vars/main.yml index c8e995146..0e56a6eac 100644 --- a/roles/openshift_logging_elasticsearch/vars/main.yml +++ b/roles/openshift_logging_elasticsearch/vars/main.yml @@ -1,6 +1,6 @@ --- -__latest_es_version: "3_6" -__allowed_es_versions: ["3_5", "3_6", "3_7"] +__latest_es_version: "3_8" +__allowed_es_versions: ["3_5", "3_6", "3_7", "3_8"] __allowed_es_types: ["data-master", "data-client", "master", "client"] __es_log_appenders: ['file', 'console'] __kibana_index_modes: ["unique", "shared_ops"] diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml index 08d7561ac..486cfb8bc 100644 --- a/roles/openshift_logging_fluentd/tasks/main.yaml +++ b/roles/openshift_logging_fluentd/tasks/main.yaml @@ -108,38 +108,28 @@ dest: "{{ tempdir }}/fluent.conf" vars: deploy_type: "{{ openshift_logging_fluentd_deployment_type }}" - when: fluentd_config_contents is undefined - changed_when: no - copy: src: fluentd-throttle-config.yaml dest: "{{ tempdir }}/fluentd-throttle-config.yaml" - when: fluentd_throttle_contents is undefined - changed_when: no - copy: src: secure-forward.conf dest: "{{ tempdir }}/secure-forward.conf" - when: fluentd_secureforward_contents is undefined - changed_when: no - -- copy: - content: "{{ fluentd_config_contents }}" - dest: "{{ tempdir }}/fluent.conf" - when: fluentd_config_contents is defined - changed_when: no -- copy: - content: "{{ fluentd_throttle_contents }}" - dest: "{{ tempdir }}/fluentd-throttle-config.yaml" - when: fluentd_throttle_contents is defined - changed_when: no - -- copy: - content: "{{ fluentd_secureforward_contents }}" - dest: "{{ tempdir }}/secure-forward.conf" - when: fluentd_secureforward_contents is defined - changed_when: no +- include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-fluentd" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "fluent.conf" + new_file: "{{ tempdir }}/fluent.conf" + - current_file: "throttle-config.yaml" + new_file: "{{ tempdir }}/fluentd-throttle-config.yaml" + - current_file: "secure-forward.conf" + new_file: "{{ tempdir }}/secure-forward.conf" - name: Set Fluentd configmap oc_configmap: diff --git a/roles/openshift_logging_fluentd/vars/main.yml b/roles/openshift_logging_fluentd/vars/main.yml index 92a426952..762e3d4d0 100644 --- a/roles/openshift_logging_fluentd/vars/main.yml +++ b/roles/openshift_logging_fluentd/vars/main.yml @@ -1,5 +1,5 @@ --- -__latest_fluentd_version: "3_6" -__allowed_fluentd_versions: ["3_5", "3_6", "3_7"] +__latest_fluentd_version: "3_8" +__allowed_fluentd_versions: ["3_5", "3_6", "3_7", "3_8"] __allowed_fluentd_types: ["hosted", "secure-aggregator", "secure-host"] __allowed_mux_client_modes: ["minimal", "maximal"] diff --git a/roles/openshift_logging_kibana/vars/main.yml b/roles/openshift_logging_kibana/vars/main.yml index 241877a02..a2c54d8e4 100644 --- a/roles/openshift_logging_kibana/vars/main.yml +++ b/roles/openshift_logging_kibana/vars/main.yml @@ -1,3 +1,3 @@ --- -__latest_kibana_version: "3_6" -__allowed_kibana_versions: ["3_5", "3_6", "3_7"] +__latest_kibana_version: "3_8" +__allowed_kibana_versions: ["3_5", "3_6", "3_7", "3_8"] diff --git a/roles/openshift_logging_mux/tasks/main.yaml b/roles/openshift_logging_mux/tasks/main.yaml index 59a6301d7..a281c6a53 100644 --- a/roles/openshift_logging_mux/tasks/main.yaml +++ b/roles/openshift_logging_mux/tasks/main.yaml @@ -88,26 +88,24 @@ - copy: src: fluent.conf dest: "{{mktemp.stdout}}/fluent-mux.conf" - when: fluentd_mux_config_contents is undefined changed_when: no - copy: src: secure-forward.conf dest: "{{mktemp.stdout}}/secure-forward-mux.conf" - when: fluentd_mux_securefoward_contents is undefined changed_when: no -- copy: - content: "{{fluentd_mux_config_contents}}" - dest: "{{mktemp.stdout}}/fluent-mux.conf" - when: fluentd_mux_config_contents is defined - changed_when: no - -- copy: - content: "{{fluentd_mux_secureforward_contents}}" - dest: "{{mktemp.stdout}}/secure-forward-mux.conf" - when: fluentd_mux_secureforward_contents is defined - changed_when: no +- include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-mux" + configmap_namespace: "{{ openshift_logging_mux_namespace }}" + configmap_file_names: + - current_file: "fluent.conf" + new_file: "{{ tempdir }}/fluent-mux.conf" + - current_file: "secure-forward.conf" + new_file: "{{ tempdir }}/secure-forward-mux.conf" - name: Set Mux configmap oc_configmap: diff --git a/roles/openshift_logging_mux/vars/main.yml b/roles/openshift_logging_mux/vars/main.yml index e7b57f4b5..1da053b4a 100644 --- a/roles/openshift_logging_mux/vars/main.yml +++ b/roles/openshift_logging_mux/vars/main.yml @@ -1,3 +1,3 @@ --- -__latest_mux_version: "3_6" -__allowed_mux_versions: ["3_5", "3_6", "3_7"] +__latest_mux_version: "3_8" +__allowed_mux_versions: ["3_5", "3_6", "3_7", "3_8"] diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index eb362816a..103572291 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -99,7 +99,3 @@ - include_tasks: config/workaround-bz1331590-ovs-oom-fix.yml when: openshift_node_use_openshift_sdn | default(true) | bool - -- name: include bootstrap node config - include_tasks: bootstrap.yml - when: openshift_node_bootstrap diff --git a/roles/openshift_sanitize_inventory/meta/main.yml b/roles/openshift_sanitize_inventory/meta/main.yml index 324ba06d8..cde3eccb6 100644 --- a/roles/openshift_sanitize_inventory/meta/main.yml +++ b/roles/openshift_sanitize_inventory/meta/main.yml @@ -14,3 +14,4 @@ galaxy_info: - system dependencies: - role: lib_utils +- role: lib_openshift |