diff options
127 files changed, 2682 insertions, 2613 deletions
@@ -30,10 +30,9 @@ Setup - [How to build the openshift-ansible rpms](BUILD.md) - Directory Structure: - - [cloud.rb](cloud.rb) - light wrapper around Ansible - [bin/cluster](bin/cluster) - python script to easily create OpenShift 3 clusters + - [docs](docs) - Documentation for the project - [filter_plugins/](filter_plugins) - custom filters used to manipulate data in Ansible - [inventory/](inventory) - houses Ansible dynamic inventory scripts - - [lib/](lib) - library components of cloud.rb - [playbooks/](playbooks) - houses host-type Ansible playbooks (launch, config, destroy, vars) - [roles/](roles) - shareable Ansible tasks diff --git a/README_AWS.md b/README_AWS.md index dc93357ee..7f4b1832b 100644 --- a/README_AWS.md +++ b/README_AWS.md @@ -18,7 +18,7 @@ Create a credentials file ``` source ~/.aws_creds ``` -Note: You must source this file in each shell that you want to run cloud.rb +Note: You must source this file before running any Ansible commands. (Optional) Setup your $HOME/.ssh/config file diff --git a/README_OSE.md b/README_OSE.md index 41a6f2935..dffabc714 100644 --- a/README_OSE.md +++ b/README_OSE.md @@ -80,7 +80,7 @@ ansible_ssh_user=root deployment_type=enterprise # Pre-release registry URL -openshift_registry_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version} +oreg_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version} # Pre-release additional repo openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', diff --git a/ansible.cfg b/ansible.cfg.example index 6a7722ad8..6a7722ad8 100644 --- a/ansible.cfg +++ b/ansible.cfg.example @@ -17,13 +17,10 @@ from openshift_ansible.awsutil import ArgumentError CONFIG_MAIN_SECTION = 'main' CONFIG_HOST_TYPE_ALIAS_SECTION = 'host_type_aliases' -CONFIG_INVENTORY_OPTION = 'inventory' - class Ohi(object): def __init__(self): - self.inventory = None self.host_type_aliases = {} self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) @@ -35,7 +32,7 @@ class Ohi(object): self.parse_cli_args() self.parse_config_file() - self.aws = awsutil.AwsUtil(self.inventory, self.host_type_aliases) + self.aws = awsutil.AwsUtil(self.host_type_aliases) def run(self): if self.args.list_host_types: @@ -47,12 +44,12 @@ class Ohi(object): self.args.env is not None: # Both env and host-type specified hosts = self.aws.get_host_list(host_type=self.args.host_type, \ - env=self.args.env) + envs=self.args.env) if self.args.host_type is None and \ self.args.env is not None: # Only env specified - hosts = self.aws.get_host_list(env=self.args.env) + hosts = self.aws.get_host_list(envs=self.args.env) if self.args.host_type is not None and \ self.args.env is None: @@ -76,10 +73,6 @@ class Ohi(object): config = ConfigParser.ConfigParser() config.read(self.config_path) - if config.has_section(CONFIG_MAIN_SECTION) and \ - config.has_option(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION): - self.inventory = config.get(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION) - self.host_type_aliases = {} if config.has_section(CONFIG_HOST_TYPE_ALIAS_SECTION): for alias in config.options(CONFIG_HOST_TYPE_ALIAS_SECTION): diff --git a/bin/openshift-ansible-bin.spec b/bin/openshift-ansible-bin.spec index 29aaff9ae..884d4eb0a 100644 --- a/bin/openshift-ansible-bin.spec +++ b/bin/openshift-ansible-bin.spec @@ -1,6 +1,6 @@ Summary: OpenShift Ansible Scripts for working with metadata hosts Name: openshift-ansible-bin -Version: 0.0.12 +Version: 0.0.17 Release: 1%{?dist} License: ASL 2.0 URL: https://github.com/openshift/openshift-ansible @@ -24,7 +24,13 @@ mkdir -p %{buildroot}/etc/bash_completion.d mkdir -p %{buildroot}/etc/openshift_ansible cp -p ossh oscp opssh opscp ohi %{buildroot}%{_bindir} -cp -p openshift_ansible/* %{buildroot}%{python_sitelib}/openshift_ansible +cp -pP openshift_ansible/* %{buildroot}%{python_sitelib}/openshift_ansible + +# Make it so we can load multi_ec2.py as a library. +rm %{buildroot}%{python_sitelib}/openshift_ansible/multi_ec2.py* +ln -sf /usr/share/ansible/inventory/multi_ec2.py %{buildroot}%{python_sitelib}/openshift_ansible/multi_ec2.py +ln -sf /usr/share/ansible/inventory/multi_ec2.pyc %{buildroot}%{python_sitelib}/openshift_ansible/multi_ec2.pyc + cp -p ossh_bash_completion %{buildroot}/etc/bash_completion.d cp -p openshift_ansible.conf.example %{buildroot}/etc/openshift_ansible/openshift_ansible.conf @@ -36,6 +42,15 @@ cp -p openshift_ansible.conf.example %{buildroot}/etc/openshift_ansible/openshif %config(noreplace) /etc/openshift_ansible/ %changelog +* Fri May 15 2015 Thomas Wiest <twiest@redhat.com> 0.0.17-1 +- fixed the openshift-ansible-bin build (twiest@redhat.com) + +* Fri May 15 2015 Thomas Wiest <twiest@redhat.com> 0.0.14-1 +- Command line tools import multi_ec2 as lib (kwoodson@redhat.com) +- Adding cache location for multi ec2 (kwoodson@redhat.com) +* Thu May 07 2015 Thomas Wiest <twiest@redhat.com> 0.0.13-1 +- added '-e all' to ohi and fixed pylint errors. (twiest@redhat.com) + * Tue May 05 2015 Thomas Wiest <twiest@redhat.com> 0.0.12-1 - fixed opssh and opscp to allow just environment or just host-type. (twiest@redhat.com) diff --git a/bin/openshift_ansible/awsutil.py b/bin/openshift_ansible/awsutil.py index 65b269930..9df034f57 100644 --- a/bin/openshift_ansible/awsutil.py +++ b/bin/openshift_ansible/awsutil.py @@ -1,113 +1,120 @@ # vim: expandtab:tabstop=4:shiftwidth=4 -import subprocess +"""This module comprises Aws specific utility functions.""" + import os -import json import re +from openshift_ansible import multi_ec2 class ArgumentError(Exception): + """This class is raised when improper arguments are passed.""" + def __init__(self, message): + """Initialize an ArgumentError. + + Keyword arguments: + message -- the exact error message being raised + """ + super(ArgumentError, self).__init__() self.message = message class AwsUtil(object): - def __init__(self, inventory_path=None, host_type_aliases={}): - self.host_type_aliases = host_type_aliases - self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) + """This class contains the AWS utility functions.""" - if inventory_path is None: - inventory_path = os.path.realpath(os.path.join(self.file_path, \ - '..', '..', 'inventory', \ - 'multi_ec2.py')) + def __init__(self, host_type_aliases=None): + """Initialize the AWS utility class. - if not os.path.isfile(inventory_path): - raise Exception("Inventory file not found [%s]" % inventory_path) + Keyword arguments: + host_type_aliases -- a list of aliases to common host-types (e.g. ex-node) + """ + + host_type_aliases = host_type_aliases or {} + + self.host_type_aliases = host_type_aliases + self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) - self.inventory_path = inventory_path self.setup_host_type_alias_lookup() def setup_host_type_alias_lookup(self): + """Sets up the alias to host-type lookup table.""" self.alias_lookup = {} for key, values in self.host_type_aliases.iteritems(): for value in values: self.alias_lookup[value] = key + @staticmethod + def get_inventory(args=None): + """Calls the inventory script and returns a dictionary containing the inventory." - - def get_inventory(self,args=[]): - cmd = [self.inventory_path] - - if args: - cmd.extend(args) - - env = os.environ - - p = subprocess.Popen(cmd, stderr=subprocess.PIPE, - stdout=subprocess.PIPE, env=env) - - out,err = p.communicate() - - if p.returncode != 0: - raise RuntimeError(err) - - return json.loads(out.strip()) + Keyword arguments: + args -- optional arguments to pass to the inventory script + """ + mec2 = multi_ec2.MultiEc2(args) + mec2.run() + return mec2.result def get_environments(self): + """Searches for env tags in the inventory and returns all of the envs found.""" pattern = re.compile(r'^tag_environment_(.*)') envs = [] inv = self.get_inventory() for key in inv.keys(): - m = pattern.match(key) - if m: - envs.append(m.group(1)) + matched = pattern.match(key) + if matched: + envs.append(matched.group(1)) envs.sort() return envs def get_host_types(self): + """Searches for host-type tags in the inventory and returns all host-types found.""" pattern = re.compile(r'^tag_host-type_(.*)') host_types = [] inv = self.get_inventory() for key in inv.keys(): - m = pattern.match(key) - if m: - host_types.append(m.group(1)) + matched = pattern.match(key) + if matched: + host_types.append(matched.group(1)) host_types.sort() return host_types def get_security_groups(self): + """Searches for security_groups in the inventory and returns all SGs found.""" pattern = re.compile(r'^security_group_(.*)') groups = [] inv = self.get_inventory() for key in inv.keys(): - m = pattern.match(key) - if m: - groups.append(m.group(1)) + matched = pattern.match(key) + if matched: + groups.append(matched.group(1)) groups.sort() return groups - def build_host_dict_by_env(self, args=[]): + def build_host_dict_by_env(self, args=None): + """Searches the inventory for hosts in an env and returns their hostvars.""" + args = args or [] inv = self.get_inventory(args) inst_by_env = {} - for dns, host in inv['_meta']['hostvars'].items(): + for _, host in inv['_meta']['hostvars'].items(): # If you don't have an environment tag, we're going to ignore you if 'ec2_tag_environment' not in host: continue if host['ec2_tag_environment'] not in inst_by_env: inst_by_env[host['ec2_tag_environment']] = {} - host_id = "%s:%s" % (host['ec2_tag_Name'],host['ec2_id']) + host_id = "%s:%s" % (host['ec2_tag_Name'], host['ec2_id']) inst_by_env[host['ec2_tag_environment']][host_id] = host return inst_by_env - # Display host_types def print_host_types(self): + """Gets the list of host types and aliases and outputs them in columns.""" host_types = self.get_host_types() ht_format_str = "%35s" alias_format_str = "%-20s" @@ -117,22 +124,31 @@ class AwsUtil(object): print combined_format_str % ('Host Types', 'Aliases') print combined_format_str % ('----------', '-------') - for ht in host_types: + for host_type in host_types: aliases = [] - if ht in self.host_type_aliases: - aliases = self.host_type_aliases[ht] - print combined_format_str % (ht, ", ".join(aliases)) + if host_type in self.host_type_aliases: + aliases = self.host_type_aliases[host_type] + print combined_format_str % (host_type, ", ".join(aliases)) else: - print ht_format_str % ht + print ht_format_str % host_type print - # Convert host-type aliases to real a host-type def resolve_host_type(self, host_type): + """Converts a host-type alias into a host-type. + + Keyword arguments: + host_type -- The alias or host_type to look up. + + Example (depends on aliases defined in config file): + host_type = ex-node + returns: openshift-node + """ if self.alias_lookup.has_key(host_type): return self.alias_lookup[host_type] return host_type - def gen_env_tag(self, env): + @staticmethod + def gen_env_tag(env): """Generate the environment tag """ return "tag_environment_%s" % env @@ -149,28 +165,44 @@ class AwsUtil(object): host_type = self.resolve_host_type(host_type) return "tag_env-host-type_%s-%s" % (env, host_type) - def get_host_list(self, host_type=None, env=None): + def get_host_list(self, host_type=None, envs=None): """Get the list of hosts from the inventory using host-type and environment """ + envs = envs or [] inv = self.get_inventory() - if host_type is not None and \ - env is not None: - # Both host type and environment were specified - env_host_type_tag = self.gen_env_host_type_tag(host_type, env) - return inv[env_host_type_tag] + # We prefer to deal with a list of environments + if issubclass(type(envs), basestring): + if envs == 'all': + envs = self.get_environments() + else: + envs = [envs] - if host_type is None and \ - env is not None: + if host_type and envs: + # Both host type and environment were specified + retval = [] + for env in envs: + env_host_type_tag = self.gen_env_host_type_tag(host_type, env) + if env_host_type_tag in inv.keys(): + retval += inv[env_host_type_tag] + return set(retval) + + if envs and not host_type: # Just environment was specified - host_type_tag = self.gen_env_tag(env) - return inv[host_type_tag] - - if host_type is not None and \ - env is None: + retval = [] + for env in envs: + env_tag = AwsUtil.gen_env_tag(env) + if env_tag in inv.keys(): + retval += inv[env_tag] + return set(retval) + + if host_type and not envs: # Just host-type was specified + retval = [] host_type_tag = self.gen_host_type_tag(host_type) - return inv[host_type_tag] + if host_type_tag in inv.keys(): + retval = inv[host_type_tag] + return set(retval) # We should never reach here! raise ArgumentError("Invalid combination of parameters") diff --git a/bin/openshift_ansible/multi_ec2.py b/bin/openshift_ansible/multi_ec2.py new file mode 120000 index 000000000..660a0418e --- /dev/null +++ b/bin/openshift_ansible/multi_ec2.py @@ -0,0 +1 @@ +../../inventory/multi_ec2.py
\ No newline at end of file @@ -11,11 +11,9 @@ import ConfigParser from openshift_ansible import awsutil CONFIG_MAIN_SECTION = 'main' -CONFIG_INVENTORY_OPTION = 'inventory' class Oscp(object): def __init__(self): - self.inventory = None self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) # Default the config path to /etc @@ -29,13 +27,13 @@ class Oscp(object): # parse host and user self.process_host() - self.aws = awsutil.AwsUtil(self.inventory) + self.aws = awsutil.AwsUtil() # get a dict of host inventory - if self.args.list: - self.get_hosts() - else: + if self.args.refresh_cache: self.get_hosts(True) + else: + self.get_hosts() if (self.args.src == '' or self.args.dest == '') and not self.args.list: self.parser.print_help() @@ -56,10 +54,6 @@ class Oscp(object): config = ConfigParser.ConfigParser() config.read(self.config_path) - if config.has_section(CONFIG_MAIN_SECTION) and \ - config.has_option(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION): - self.inventory = config.get(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION) - def parse_cli_args(self): parser = argparse.ArgumentParser(description='Openshift Online SSH Tool.') parser.add_argument('-e', '--env', @@ -68,6 +62,8 @@ class Oscp(object): action="store_true", help="debug mode") parser.add_argument('-v', '--verbose', default=False, action="store_true", help="Verbose?") + parser.add_argument('--refresh-cache', default=False, + action="store_true", help="Force a refresh on the host cache.") parser.add_argument('--list', default=False, action="store_true", help="list out hosts") parser.add_argument('-r', '--recurse', action='store_true', default=False, @@ -119,14 +115,14 @@ class Oscp(object): else: self.env = None - def get_hosts(self, cache_only=False): + def get_hosts(self, refresh_cache=False): '''Query our host inventory and return a dict where the format equals: dict['environment'] = [{'servername' : {}}, ] ''' - if cache_only: - self.host_inventory = self.aws.build_host_dict_by_env(['--cache-only']) + if refresh_cache: + self.host_inventory = self.aws.build_host_dict_by_env(['--refresh-cache']) else: self.host_inventory = self.aws.build_host_dict_by_env() @@ -11,11 +11,9 @@ import ConfigParser from openshift_ansible import awsutil CONFIG_MAIN_SECTION = 'main' -CONFIG_INVENTORY_OPTION = 'inventory' class Ossh(object): def __init__(self): - self.inventory = None self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) # Default the config path to /etc @@ -26,13 +24,12 @@ class Ossh(object): self.parse_cli_args() self.parse_config_file() - self.aws = awsutil.AwsUtil(self.inventory) + self.aws = awsutil.AwsUtil() - # get a dict of host inventory - if self.args.list: - self.get_hosts() - else: + if self.args.refresh_cache: self.get_hosts(True) + else: + self.get_hosts() # parse host and user self.process_host() @@ -55,10 +52,6 @@ class Ossh(object): config = ConfigParser.ConfigParser() config.read(self.config_path) - if config.has_section(CONFIG_MAIN_SECTION) and \ - config.has_option(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION): - self.inventory = config.get(CONFIG_MAIN_SECTION, CONFIG_INVENTORY_OPTION) - def parse_cli_args(self): parser = argparse.ArgumentParser(description='Openshift Online SSH Tool.') parser.add_argument('-e', '--env', action="store", @@ -67,6 +60,8 @@ class Ossh(object): action="store_true", help="debug mode") parser.add_argument('-v', '--verbose', default=False, action="store_true", help="Verbose?") + parser.add_argument('--refresh-cache', default=False, + action="store_true", help="Force a refresh on the host cache.") parser.add_argument('--list', default=False, action="store_true", help="list out hosts") parser.add_argument('-c', '--command', action='store', @@ -109,14 +104,14 @@ class Ossh(object): if self.args.login_name: self.user = self.args.login_name - def get_hosts(self, cache_only=False): + def get_hosts(self, refresh_cache=False): '''Query our host inventory and return a dict where the format equals: dict['servername'] = dns_name ''' - if cache_only: - self.host_inventory = self.aws.build_host_dict_by_env(['--cache-only']) + if refresh_cache: + self.host_inventory = self.aws.build_host_dict_by_env(['--refresh-cache']) else: self.host_inventory = self.aws.build_host_dict_by_env() diff --git a/cloud.rb b/cloud.rb deleted file mode 100755 index 934066662..000000000 --- a/cloud.rb +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env ruby - -require 'thor' -require_relative 'lib/gce_command' -require_relative 'lib/aws_command' - -# Don't buffer output to the client -STDOUT.sync = true -STDERR.sync = true - -module OpenShift - module Ops - class CloudCommand < Thor - desc 'gce', 'Manages Google Compute Engine assets' - subcommand "gce", GceCommand - - desc 'aws', 'Manages Amazon Web Services assets' - subcommand "aws", AwsCommand - end - end -end - -if __FILE__ == $0 - SCRIPT_DIR = File.expand_path(File.dirname(__FILE__)) - Dir.chdir(SCRIPT_DIR) do - # Kick off thor - OpenShift::Ops::CloudCommand.start(ARGV) - end -end diff --git a/docs/best_practices_guide.adoc b/docs/best_practices_guide.adoc new file mode 100644 index 000000000..301c6ccda --- /dev/null +++ b/docs/best_practices_guide.adoc @@ -0,0 +1,153 @@ +// vim: ft=asciidoc + += Openshift-Ansible Best Practices Guide + +The purpose of this guide is to describe the preferred patterns and best practices used in this repository (both in ansible and python). + +It is important to note that this repository may not currently comply with all best practices, but the intention is that it will. + +All new pull requests created against this repository MUST comply with this guide. + +This guide complies with https://www.ietf.org/rfc/rfc2119.txt[RFC2119]. + + +== Pull Requests + +[cols="2v,v"] +|=== +| **Rule** +| All pull requests MUST pass the build bot *before* they are merged. +|=== + + +The purpose of this rule is to avoid cases where the build bot will fail pull requests for code modified in a previous pull request. + +The tooling is flexible enough that exceptions can be made so that the tool the build bot is running will ignore certain areas or certain checks, but the build bot itself must pass for the pull request to be merged. + + + +== Python + +=== PyLint +http://www.pylint.org/[PyLint] is used in an attempt to keep the python code as clean and as managable as possible. The build bot runs each pull request through PyLint and any warnings or errors cause the build bot to fail the pull request. + +''' +[cols="2v,v"] +|=== +| **Rule** +| PyLint rules MUST NOT be disabled on a whole file. +|=== + +Instead, http://docs.pylint.org/faq.html#is-it-possible-to-locally-disable-a-particular-message[disable the PyLint check on the line where PyLint is complaining]. + +''' +[cols="2v,v"] +|=== +| **Rule** +| PyLint rules MUST NOT be disabled unless they meet one of the following exceptions +|=== + +.Exceptions: +1. When PyLint fails because of a dependency that can't be installed on the build bot +1. When PyLint fails because of including a module that is outside of control (like Ansible) + +''' +[cols="2v,v"] +|=== +| **Rule** +| All PyLint rule disables MUST be documented in the code. +|=== + +The purpose of this rule is to inform future developers about the disable. + +.Specifically, the following MUST accompany every PyLint disable: +1. Why is the check being disabled? +1. Is disabling this check meant to be permanent or temporary? + +.Example: +[source,python] +---- +# Reason: disable pylint maybe-no-member because overloaded use of +# the module name causes pylint to not detect that 'results' +# is an array or hash +# Status: permanently disabled unless a way is found to fix this. +# pylint: disable=maybe-no-member +metadata[line] = results.pop() +---- + + +== Ansible + +=== Roles +.Context +* http://docs.ansible.com/playbooks_best_practices.html#directory-layout[Ansible Suggested Directory Layout] + +''' +[cols="2v,v"] +|=== +| **Rule** +| The Ansible roles directory MUST maintain a flat structure. +|=== + +.The purpose of this rule is to: +* Comply with the upstream best practices +* Make it familiar for new contributors +* Make it compatible with Ansible Galaxy + +''' +[cols="2v,v"] +|=== +| **Rule** +| Ansible Roles SHOULD be named like technology_component[_subcomponent]. +|=== + +For clarity, it is suggested to follow a pattern when naming roles. It is important to note that this is a recommendation for role naming, and follows the pattern used by upstream. + +Many times the `technology` portion of the pattern will line up with a package name. It is advised that whenever possible, the package name should be used. + +.Examples: +* The role to configure an OpenShift Master is called `openshift_master` +* The role to configure OpenShift specific yum repositories is called `openshift_repos` + +=== Filters +.Context: +* https://docs.ansible.com/playbooks_filters.html[Ansible Playbook Filters] +* http://jinja.pocoo.org/docs/dev/templates/#builtin-filters[Jinja2 Builtin Filters] + +''' +[cols="2v,v"] +|=== +| **Rule** +| The `default` filter SHOULD replace empty strings, lists, etc. +|=== + +When using the jinja2 `default` filter, unless the variable is a boolean, specify `true` as the second parameter. This will cause the default filter to replace empty strings, lists, etc with the provided default. + +This is because it is preferable to either have a sane default set than to have an empty string, list, etc. For example, it is preferable to have a config value set to a sane default than to have it simply set as an empty string. + +.From the http://jinja.pocoo.org/docs/dev/templates/[Jinja2 Docs]: +[quote] +If you want to use default with variables that evaluate to false you have to set the second parameter to true + +.Example: +[source,yaml] +---- +--- +- hosts: localhost + gather_facts: no + vars: + somevar: '' + tasks: + - debug: var=somevar + + - name: "Will output 'somevar: []'" + debug: "msg='somevar: [{{ somevar | default('the string was empty') }}]'" + + - name: "Will output 'somevar: [the string was empty]'" + debug: "msg='somevar: [{{ somevar | default('the string was empty', true) }}]'" +---- + + +In other words, normally the `default` filter will only replace the value if it's undefined. By setting the second parameter to `true`, it will also replace the value if it defaults to a false value in python, so None, empty list, empty string, etc. + +This is almost always more desirable than an empty list, string, etc. diff --git a/docs/core_concepts_guide.adoc b/docs/core_concepts_guide.adoc new file mode 100644 index 000000000..38187c55e --- /dev/null +++ b/docs/core_concepts_guide.adoc @@ -0,0 +1,43 @@ +// vim: ft=asciidoc + += Openshift-Ansible Core Concepts Guide + +The purpose of this guide is to describe core concepts used in this repository. + +It is important to note that this repository may not currently implement all of the concepts, but the intention is that it will. + +== Logical Grouping Concepts +The following are the concepts used to logically group OpenShift cluster instances. + +These groupings are used to perform operations specifically against instances in the specified group. + +For example, run an Ansible playbook against all instances in the `production` environment, or run an adhoc command against all instances in the `acme-corp` cluster group. + +=== Cluster +A Cluster is a complete install of OpenShift (master, nodes, registry, router, etc). + +Example: Acme Corp has sales and marketing departments that both want to use OpenShift for their internal applications, but they do not want to share resources because they have different cost centers. Each department could have their own completely separate install of OpenShift. Each install is a separate OpenShift cluster. + +Defined Clusters: +`acme-sales` +`acme-marketing` + +=== Cluster Group +A cluster group is a logical grouping of one or more clusters. Which clusters are in which cluster groups is determined by the OpenShift administrators. + +Example: Extending the example above, both marketing and sales clusters are part of Acme Corp. Let's say that Acme Corp contracts with Hosting Corp to host their OpenShift clusters. Hosting Corp could create an Acme Corp cluster group. + +This would logically group Acme Corp resources from other Hosting Corp customers, which would enable the Hosting Corp's OpenShift administrators to run operations specifically targeting Acme Corp instances. + +Defined Cluster Group: +`acme-corp` + +=== Environment +An environment is a logical grouping of one or more cluster groups. How the environment is defined is determined by the OpenShift administrators. + +Example: Extending the two examples above, Hosting Corp is upgrading to the latest version of OpenShift. Before deploying it to their clusters in the Production environment, they want to test it out. So, Hosting Corp runs an Ansible playbook specifically against all of the cluster groups in the Staging environment in order to do the OpenShift upgrade. + + +Defined Environments: +`production` +`staging` diff --git a/docs/style_guide.adoc b/docs/style_guide.adoc new file mode 100644 index 000000000..3b888db12 --- /dev/null +++ b/docs/style_guide.adoc @@ -0,0 +1,138 @@ +// vim: ft=asciidoc + += Openshift-Ansible Style Guide + +The purpose of this guide is to describe the preferred coding conventions used in this repository (both in ansible and python). + +It is important to note that this repository may not currently comply with all style guide rules, but the intention is that it will. + +All new pull requests created against this repository MUST comply with this guide. + +This style guide complies with https://www.ietf.org/rfc/rfc2119.txt[RFC2119]. + +== Python + + +=== Python Maximum Line Length + +.Context: +* https://www.python.org/dev/peps/pep-0008/#maximum-line-length[Python Pep8 Line Length] + +''' +[cols="2v,v"] +|=== +| **Rule** +| All lines SHOULD be no longer than 80 characters. +|=== + +Every attempt SHOULD be made to comply with this soft line length limit, and only when it makes the code more readable should this be violated. + +Code readability is subjective, therefore pull-requests SHOULD still be merged, even if they violate this soft limit as it is up to the individual contributor to determine if they should violate the 80 character soft limit. + + +''' +[cols="2v,v"] +|=== +| **Rule** +| All lines MUST be no longer than 120 characters. +|=== + +This is a hard limit and is enforced by the build bot. This check MUST NOT be disabled. + + + +== Ansible + +=== Ansible Global Variables +Ansible global variables are defined as any variables outside of ansible roles. Examples include playbook variables, variables passed in on the cli, etc. + +''' +[cols="2v,v"] +|=== +| **Rule** +| Global variables MUST have a prefix of g_ +|=== + + +Example: +[source] +---- +g_environment: someval +---- + +=== Ansible Role Variables +Ansible role variables are defined as variables contained in (or passed into) a role. + +''' +[cols="2v,v"] +|=== +| **Rule** +| Role variables MUST have a prefix of atleast 3 characters. See below for specific naming rules. +|=== + +==== Role with 3 (or more) words in the name + +Take the first letter of each of the words. + +.3 word example: +* Role name: made_up_role +* Prefix: mur +[source] +---- +mur_var1: value_one +---- + +.4 word example: +* Role name: totally_made_up_role +* Prefix: tmur +[source] +---- +tmur_var1: value_one +---- + + + +==== Role with 2 (or less) words in the name + +Make up a prefix that makes sense. + +.1 word example: +* Role name: ansible +* Prefix: ans +[source] +---- +ans_var1: value_one +---- + +.2 word example: +* Role name: ansible_tower +* Prefix: tow +[source] +---- +tow_var1: value_one +---- + + +==== Role name prefix conflicts +If two role names contain words that start with the same letters, it will seem like their prefixes would conflict. + +Role variables are confined to the roles themselves, so this is actually only a problem if one of the roles depends on the other role (or uses includes into the other role). + +.Same prefix example: +* First Role Name: made_up_role +* First Role Prefix: mur +* Second Role Name: my_uber_role +* Second Role Prefix: mur +[source] +---- +- hosts: localhost + roles: + - { role: made_up_role, mur_var1: val1 } + - { role: my_uber_role, mur_var1: val2 } +---- + +Even though both roles have the same prefix (mur), and even though both roles have a variable named mur_var1, these two variables never exist outside of their respective roles. This means that this is not a problem. + +This would only be a problem if my_uber_role depended on made_up_role, or vice versa. Or if either of these two roles included things from the other. + +This is enough of a corner case that it is unlikely to happen. If it does, it will be addressed on a case by case basis. diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py index 097038450..33d5e6cc3 100644 --- a/filter_plugins/oo_filters.py +++ b/filter_plugins/oo_filters.py @@ -9,188 +9,210 @@ from ansible import errors from operator import itemgetter import pdb -def oo_pdb(arg): - ''' This pops you into a pdb instance where arg is the data passed in - from the filter. - Ex: "{{ hostvars | oo_pdb }}" - ''' - pdb.set_trace() - return arg - -def oo_len(arg): - ''' This returns the length of the argument - Ex: "{{ hostvars | oo_len }}" - ''' - return len(arg) - -def get_attr(data, attribute=None): - ''' This looks up dictionary attributes of the form a.b.c and returns - the value. - Ex: data = {'a': {'b': {'c': 5}}} - attribute = "a.b.c" - returns 5 - ''' - if not attribute: - raise errors.AnsibleFilterError("|failed expects attribute to be set") - - ptr = data - for attr in attribute.split('.'): - ptr = ptr[attr] - - return ptr - -def oo_flatten(data): - ''' This filter plugin will flatten a list of lists - ''' - if not issubclass(type(data), list): - raise errors.AnsibleFilterError("|failed expects to flatten a List") - - return [item for sublist in data for item in sublist] - - -def oo_collect(data, attribute=None, filters=None): - ''' This takes a list of dict and collects all attributes specified into a - list If filter is specified then we will include all items that match - _ALL_ of filters. - Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return - {'a':2, 'z': 'z'}, # True, return - {'a':3, 'z': 'z'}, # True, return - {'a':4, 'z': 'b'}, # FAILED, obj['z'] != obj['z'] - ] - attribute = 'a' - filters = {'z': 'z'} - returns [1, 2, 3] - ''' - if not issubclass(type(data), list): - raise errors.AnsibleFilterError("|failed expects to filter on a List") - - if not attribute: - raise errors.AnsibleFilterError("|failed expects attribute to be set") - - if filters is not None: - if not issubclass(type(filters), dict): - raise errors.AnsibleFilterError("|fialed expects filter to be a" - " dict") - retval = [get_attr(d, attribute) for d in data if ( - all([d[key] == filters[key] for key in filters]))] - else: - retval = [get_attr(d, attribute) for d in data] - - return retval - -def oo_select_keys(data, keys): - ''' This returns a list, which contains the value portions for the keys - Ex: data = { 'a':1, 'b':2, 'c':3 } - keys = ['a', 'c'] - returns [1, 3] - ''' - - if not issubclass(type(data), dict): - raise errors.AnsibleFilterError("|failed expects to filter on a dict") - - if not issubclass(type(keys), list): - raise errors.AnsibleFilterError("|failed expects first param is a list") - - # Gather up the values for the list of keys passed in - retval = [data[key] for key in keys] - - return retval - -def oo_prepend_strings_in_list(data, prepend): - ''' This takes a list of strings and prepends a string to each item in the - list - Ex: data = ['cart', 'tree'] - prepend = 'apple-' - returns ['apple-cart', 'apple-tree'] - ''' - if not issubclass(type(data), list): - raise errors.AnsibleFilterError("|failed expects first param is a list") - if not all(isinstance(x, basestring) for x in data): - raise errors.AnsibleFilterError("|failed expects first param is a list" - " of strings") - retval = [prepend + s for s in data] - return retval - -def oo_ami_selector(data, image_name): - ''' This takes a list of amis and an image name and attempts to return - the latest ami. - ''' - if not issubclass(type(data), list): - raise errors.AnsibleFilterError("|failed expects first param is a list") - - if not data: - return None - else: - if image_name is None or not image_name.endswith('_*'): - ami = sorted(data, key=itemgetter('name'), reverse=True)[0] - return ami['ami_id'] + +class FilterModule(object): + ''' Custom ansible filters ''' + + @staticmethod + def oo_pdb(arg): + ''' This pops you into a pdb instance where arg is the data passed in + from the filter. + Ex: "{{ hostvars | oo_pdb }}" + ''' + pdb.set_trace() + return arg + + @staticmethod + def oo_len(arg): + ''' This returns the length of the argument + Ex: "{{ hostvars | oo_len }}" + ''' + return len(arg) + + @staticmethod + def get_attr(data, attribute=None): + ''' This looks up dictionary attributes of the form a.b.c and returns + the value. + Ex: data = {'a': {'b': {'c': 5}}} + attribute = "a.b.c" + returns 5 + ''' + if not attribute: + raise errors.AnsibleFilterError("|failed expects attribute to be set") + + ptr = data + for attr in attribute.split('.'): + ptr = ptr[attr] + + return ptr + + @staticmethod + def oo_flatten(data): + ''' This filter plugin will flatten a list of lists + ''' + if not issubclass(type(data), list): + raise errors.AnsibleFilterError("|failed expects to flatten a List") + + return [item for sublist in data for item in sublist] + + + @staticmethod + def oo_collect(data, attribute=None, filters=None): + ''' This takes a list of dict and collects all attributes specified into a + list If filter is specified then we will include all items that match + _ALL_ of filters. + Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return + {'a':2, 'z': 'z'}, # True, return + {'a':3, 'z': 'z'}, # True, return + {'a':4, 'z': 'b'}, # FAILED, obj['z'] != obj['z'] + ] + attribute = 'a' + filters = {'z': 'z'} + returns [1, 2, 3] + ''' + if not issubclass(type(data), list): + raise errors.AnsibleFilterError("|failed expects to filter on a List") + + if not attribute: + raise errors.AnsibleFilterError("|failed expects attribute to be set") + + if filters is not None: + if not issubclass(type(filters), dict): + raise errors.AnsibleFilterError("|fialed expects filter to be a" + " dict") + retval = [FilterModule.get_attr(d, attribute) for d in data if ( + all([d[key] == filters[key] for key in filters]))] else: - ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data] - ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0] - return ami['ami_id'] - -def oo_ec2_volume_definition(data, host_type, docker_ephemeral=False): - ''' This takes a dictionary of volume definitions and returns a valid ec2 - volume definition based on the host_type and the values in the - dictionary. - The dictionary should look similar to this: - { 'master': - { 'root': - { 'volume_size': 10, 'device_type': 'gp2', - 'iops': 500 - } - }, - 'node': - { 'root': - { 'volume_size': 10, 'device_type': 'io1', - 'iops': 1000 + retval = [FilterModule.get_attr(d, attribute) for d in data] + + return retval + + @staticmethod + def oo_select_keys(data, keys): + ''' This returns a list, which contains the value portions for the keys + Ex: data = { 'a':1, 'b':2, 'c':3 } + keys = ['a', 'c'] + returns [1, 3] + ''' + + if not issubclass(type(data), dict): + raise errors.AnsibleFilterError("|failed expects to filter on a dict") + + if not issubclass(type(keys), list): + raise errors.AnsibleFilterError("|failed expects first param is a list") + + # Gather up the values for the list of keys passed in + retval = [data[key] for key in keys] + + return retval + + @staticmethod + def oo_prepend_strings_in_list(data, prepend): + ''' This takes a list of strings and prepends a string to each item in the + list + Ex: data = ['cart', 'tree'] + prepend = 'apple-' + returns ['apple-cart', 'apple-tree'] + ''' + if not issubclass(type(data), list): + raise errors.AnsibleFilterError("|failed expects first param is a list") + if not all(isinstance(x, basestring) for x in data): + raise errors.AnsibleFilterError("|failed expects first param is a list" + " of strings") + retval = [prepend + s for s in data] + return retval + + @staticmethod + def oo_combine_key_value(data, joiner='='): + '''Take a list of dict in the form of { 'key': 'value'} and + arrange them as a list of strings ['key=value'] + ''' + if not issubclass(type(data), list): + raise errors.AnsibleFilterError("|failed expects first param is a list") + + rval = [] + for item in data: + rval.append("%s%s%s" % (item['key'], joiner, item['value'])) + + return rval + + @staticmethod + def oo_ami_selector(data, image_name): + ''' This takes a list of amis and an image name and attempts to return + the latest ami. + ''' + if not issubclass(type(data), list): + raise errors.AnsibleFilterError("|failed expects first param is a list") + + if not data: + return None + else: + if image_name is None or not image_name.endswith('_*'): + ami = sorted(data, key=itemgetter('name'), reverse=True)[0] + return ami['ami_id'] + else: + ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data] + ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0] + return ami['ami_id'] + + @staticmethod + def oo_ec2_volume_definition(data, host_type, docker_ephemeral=False): + ''' This takes a dictionary of volume definitions and returns a valid ec2 + volume definition based on the host_type and the values in the + dictionary. + The dictionary should look similar to this: + { 'master': + { 'root': + { 'volume_size': 10, 'device_type': 'gp2', + 'iops': 500 + } }, - 'docker': - { 'volume_size': 40, 'device_type': 'gp2', - 'iops': 500, 'ephemeral': 'true' + 'node': + { 'root': + { 'volume_size': 10, 'device_type': 'io1', + 'iops': 1000 + }, + 'docker': + { 'volume_size': 40, 'device_type': 'gp2', + 'iops': 500, 'ephemeral': 'true' + } } } - } - ''' - if not issubclass(type(data), dict): - raise errors.AnsibleFilterError("|failed expects first param is a dict") - if host_type not in ['master', 'node']: - raise errors.AnsibleFilterError("|failed expects either master or node" - " host type") - - root_vol = data[host_type]['root'] - root_vol['device_name'] = '/dev/sda1' - root_vol['delete_on_termination'] = True - if root_vol['device_type'] != 'io1': - root_vol.pop('iops', None) - if host_type == 'node': - docker_vol = data[host_type]['docker'] - docker_vol['device_name'] = '/dev/xvdb' - docker_vol['delete_on_termination'] = True - if docker_vol['device_type'] != 'io1': - docker_vol.pop('iops', None) - if docker_ephemeral: - docker_vol.pop('device_type', None) - docker_vol.pop('delete_on_termination', None) - docker_vol['ephemeral'] = 'ephemeral0' - return [root_vol, docker_vol] - return [root_vol] - -# disabling pylint checks for too-few-public-methods and no-self-use since we -# need to expose a FilterModule object that has a filters method that returns -# a mapping of filter names to methods. -# pylint: disable=too-few-public-methods, no-self-use -class FilterModule(object): - ''' FilterModule ''' + ''' + if not issubclass(type(data), dict): + raise errors.AnsibleFilterError("|failed expects first param is a dict") + if host_type not in ['master', 'node']: + raise errors.AnsibleFilterError("|failed expects either master or node" + " host type") + + root_vol = data[host_type]['root'] + root_vol['device_name'] = '/dev/sda1' + root_vol['delete_on_termination'] = True + if root_vol['device_type'] != 'io1': + root_vol.pop('iops', None) + if host_type == 'node': + docker_vol = data[host_type]['docker'] + docker_vol['device_name'] = '/dev/xvdb' + docker_vol['delete_on_termination'] = True + if docker_vol['device_type'] != 'io1': + docker_vol.pop('iops', None) + if docker_ephemeral: + docker_vol.pop('device_type', None) + docker_vol.pop('delete_on_termination', None) + docker_vol['ephemeral'] = 'ephemeral0' + return [root_vol, docker_vol] + return [root_vol] + def filters(self): ''' returns a mapping of filters to methods ''' return { - "oo_select_keys": oo_select_keys, - "oo_collect": oo_collect, - "oo_flatten": oo_flatten, - "oo_len": oo_len, - "oo_pdb": oo_pdb, - "oo_prepend_strings_in_list": oo_prepend_strings_in_list, - "oo_ami_selector": oo_ami_selector, - "oo_ec2_volume_definition": oo_ec2_volume_definition + "oo_select_keys": self.oo_select_keys, + "oo_collect": self.oo_collect, + "oo_flatten": self.oo_flatten, + "oo_len": self.oo_len, + "oo_pdb": self.oo_pdb, + "oo_prepend_strings_in_list": self.oo_prepend_strings_in_list, + "oo_ami_selector": self.oo_ami_selector, + "oo_ec2_volume_definition": self.oo_ec2_volume_definition, + "oo_combine_key_value": self.oo_combine_key_value, } diff --git a/git/.pylintrc b/git/.pylintrc index 2d45f867e..af8f1656f 100644 --- a/git/.pylintrc +++ b/git/.pylintrc @@ -70,7 +70,8 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636 +# w0511 - fixme - disabled because TODOs are acceptable +disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636,W0511 [REPORTS] @@ -285,7 +286,7 @@ notes=FIXME,XXX,TODO [FORMAT] # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=120 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )?<?https?://\S+>?$ @@ -321,7 +322,7 @@ max-args=5 ignored-argument-names=_.* # Maximum number of locals for function / method body -max-locals=15 +max-locals=20 # Maximum number of return / yield for function / method body max-returns=6 diff --git a/inventory/byo/hosts b/inventory/byo/hosts index 98dbb4fd8..9a1cbce29 100644 --- a/inventory/byo/hosts +++ b/inventory/byo/hosts @@ -17,10 +17,11 @@ ansible_ssh_user=root deployment_type=enterprise # Pre-release registry URL -openshift_registry_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version} +oreg_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version} # Pre-release additional repo -openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterprise/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}] +#openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterprise/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}] +openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterpriseErrata/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}] # Origin copr repo #openshift_additional_repos=[{'id': 'openshift-origin-copr', 'name': 'OpenShift Origin COPR', 'baseurl': 'https://copr-be.cloud.fedoraproject.org/results/maxamillion/origin-next/epel-7-$basearch/', 'enabled': 1, 'gpgcheck': 1, gpgkey: 'https://copr-be.cloud.fedoraproject.org/results/maxamillion/origin-next/pubkey.gpg'}] @@ -31,4 +32,5 @@ ose3-master-ansible.test.example.com # host group for nodes [nodes] -ose3-node[1:2]-ansible.test.example.com +ose3-master-ansible.test.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default'}" +ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}" diff --git a/inventory/libvirt/hosts/libvirt_generic.py b/inventory/libvirt/hosts/libvirt_generic.py index 4652f112e..1c9c17308 100755 --- a/inventory/libvirt/hosts/libvirt_generic.py +++ b/inventory/libvirt/hosts/libvirt_generic.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 -""" +''' libvirt external inventory script ================================= @@ -12,7 +12,7 @@ To use this, copy this file over /etc/ansible/hosts and chmod +x the file. This, more or less, allows you to keep one central database containing info about all of your managed instances. -""" +''' # (c) 2015, Jason DeTiberus <jdetiber@redhat.com> # @@ -36,9 +36,7 @@ info about all of your managed instances. import argparse import ConfigParser import os -import re import sys -from time import time import libvirt import xml.etree.ElementTree as ET @@ -49,8 +47,11 @@ except ImportError: class LibvirtInventory(object): + ''' libvirt dynamic inventory ''' def __init__(self): + ''' Main execution path ''' + self.inventory = dict() # A list of groups and the hosts in that group self.cache = dict() # Details about hosts in the inventory @@ -59,13 +60,15 @@ class LibvirtInventory(object): self.parse_cli_args() if self.args.host: - print self.json_format_dict(self.get_host_info(), self.args.pretty) + print _json_format_dict(self.get_host_info(), self.args.pretty) elif self.args.list: - print self.json_format_dict(self.get_inventory(), self.args.pretty) + print _json_format_dict(self.get_inventory(), self.args.pretty) else: # default action with no options - print self.json_format_dict(self.get_inventory(), self.args.pretty) + print _json_format_dict(self.get_inventory(), self.args.pretty) def read_settings(self): + ''' Reads the settings from the libvirt.ini file ''' + config = ConfigParser.SafeConfigParser() config.read( os.path.dirname(os.path.realpath(__file__)) + '/libvirt.ini' @@ -73,6 +76,8 @@ class LibvirtInventory(object): self.libvirt_uri = config.get('libvirt', 'uri') def parse_cli_args(self): + ''' Command line argument processing ''' + parser = argparse.ArgumentParser( description='Produce an Ansible Inventory file based on libvirt' ) @@ -96,25 +101,27 @@ class LibvirtInventory(object): self.args = parser.parse_args() def get_host_info(self): + ''' Get variables about a specific host ''' + inventory = self.get_inventory() if self.args.host in inventory['_meta']['hostvars']: return inventory['_meta']['hostvars'][self.args.host] def get_inventory(self): + ''' Construct the inventory ''' + inventory = dict(_meta=dict(hostvars=dict())) conn = libvirt.openReadOnly(self.libvirt_uri) if conn is None: - print "Failed to open connection to %s" % libvirt_uri + print "Failed to open connection to %s" % self.libvirt_uri sys.exit(1) domains = conn.listAllDomains() if domains is None: - print "Failed to list domains for connection %s" % libvirt_uri + print "Failed to list domains for connection %s" % self.libvirt_uri sys.exit(1) - arp_entries = self.parse_arp_entries() - for domain in domains: hostvars = dict(libvirt_name=domain.name(), libvirt_id=domain.ID(), @@ -130,21 +137,30 @@ class LibvirtInventory(object): hostvars['libvirt_status'] = 'running' root = ET.fromstring(domain.XMLDesc()) - ns = {'ansible': 'https://github.com/ansible/ansible'} - for tag_elem in root.findall('./metadata/ansible:tags/ansible:tag', ns): + ansible_ns = {'ansible': 'https://github.com/ansible/ansible'} + for tag_elem in root.findall('./metadata/ansible:tags/ansible:tag', ansible_ns): tag = tag_elem.text - self.push(inventory, "tag_%s" % tag, domain_name) - self.push(hostvars, 'libvirt_tags', tag) + _push(inventory, "tag_%s" % tag, domain_name) + _push(hostvars, 'libvirt_tags', tag) # TODO: support more than one network interface, also support # interface types other than 'network' interface = root.find("./devices/interface[@type='network']") if interface is not None: + source_elem = interface.find('source') mac_elem = interface.find('mac') - if mac_elem is not None: - mac = mac_elem.get('address') - if mac in arp_entries: - ip_address = arp_entries[mac]['ip_address'] + if source_elem is not None and \ + mac_elem is not None: + # Adding this to disable pylint check specifically + # ignoring libvirt-python versions that + # do not include DHCPLeases + # This is needed until we upgrade the build bot to + # RHEL7 (>= 1.2.6 libvirt) + # pylint: disable=no-member + dhcp_leases = conn.networkLookupByName(source_elem.get('network')) \ + .DHCPLeases(mac_elem.get('address')) + if len(dhcp_leases) > 0: + ip_address = dhcp_leases[0]['ipaddr'] hostvars['ansible_ssh_host'] = ip_address hostvars['libvirt_ip_address'] = ip_address @@ -152,28 +168,23 @@ class LibvirtInventory(object): return inventory - def parse_arp_entries(self): - arp_entries = dict() - with open('/proc/net/arp', 'r') as f: - # throw away the header - f.readline() - - for line in f: - ip_address, _, _, mac, _, device = line.strip().split() - arp_entries[mac] = dict(ip_address=ip_address, device=device) - - return arp_entries - - def push(self, my_dict, key, element): - if key in my_dict: - my_dict[key].append(element) - else: - my_dict[key] = [element] - - def json_format_dict(self, data, pretty=False): - if pretty: - return json.dumps(data, sort_keys=True, indent=2) - else: - return json.dumps(data) +def _push(my_dict, key, element): + ''' + Push element to the my_dict[key] list. + After having initialized my_dict[key] if it dosn't exist. + ''' + + if key in my_dict: + my_dict[key].append(element) + else: + my_dict[key] = [element] + +def _json_format_dict(data, pretty=False): + ''' Serialize data to a JSON formated str ''' + + if pretty: + return json.dumps(data, sort_keys=True, indent=2) + else: + return json.dumps(data) LibvirtInventory() diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index b839a33ea..f8196aefd 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -11,9 +11,13 @@ import yaml import os import subprocess import json - +import errno +import fcntl +import tempfile +import copy CONFIG_FILE_NAME = 'multi_ec2.yaml' +DEFAULT_CACHE_PATH = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') class MultiEc2(object): ''' @@ -22,12 +26,17 @@ class MultiEc2(object): Stores a json hash of resources in result. ''' - def __init__(self): - self.args = None + def __init__(self, args=None): + # Allow args to be passed when called as a library + if not args: + self.args = {} + else: + self.args = args + + self.cache_path = DEFAULT_CACHE_PATH self.config = None self.all_ec2_results = {} self.result = {} - self.cache_path = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) same_dir_config_file = os.path.join(self.file_path, CONFIG_FILE_NAME) @@ -41,17 +50,26 @@ class MultiEc2(object): else: self.config_file = None # expect env vars - self.parse_cli_args() + def run(self): + '''This method checks to see if the local + cache is valid for the inventory. + + if the cache is valid; return cache + else the credentials are loaded from multi_ec2.yaml or from the env + and we attempt to get the inventory from the provider specified. + ''' # load yaml if self.config_file and os.path.isfile(self.config_file): self.config = self.load_yaml_config() elif os.environ.has_key("AWS_ACCESS_KEY_ID") and \ os.environ.has_key("AWS_SECRET_ACCESS_KEY"): + # Build a default config self.config = {} self.config['accounts'] = [ { 'name': 'default', + 'cache_location': DEFAULT_CACHE_PATH, 'provider': 'aws/hosts/ec2.py', 'env_vars': { 'AWS_ACCESS_KEY_ID': os.environ["AWS_ACCESS_KEY_ID"], @@ -64,11 +82,15 @@ class MultiEc2(object): else: raise RuntimeError("Could not find valid ec2 credentials in the environment.") - if self.args.refresh_cache: + # Set the default cache path but if its defined we'll assign it. + if self.config.has_key('cache_location'): + self.cache_path = self.config['cache_location'] + + if self.args.get('refresh_cache', None): self.get_inventory() self.write_to_cache() # if its a host query, fetch and do not cache - elif self.args.host: + elif self.args.get('host', None): self.get_inventory() elif not self.is_cache_valid(): # go fetch the inventories and cache them if cache is expired @@ -109,9 +131,9 @@ class MultiEc2(object): "and that it is executable. (%s)" % provider) cmds = [provider] - if self.args.host: + if self.args.get('host', None): cmds.append("--host") - cmds.append(self.args.host) + cmds.append(self.args.get('host', None)) else: cmds.append('--list') @@ -119,6 +141,54 @@ class MultiEc2(object): return subprocess.Popen(cmds, stderr=subprocess.PIPE, \ stdout=subprocess.PIPE, env=env) + + @staticmethod + def generate_config(config_data): + """Generate the ec2.ini file in as a secure temp file. + Once generated, pass it to the ec2.py as an environment variable. + """ + fildes, tmp_file_path = tempfile.mkstemp(prefix='multi_ec2.ini.') + for section, values in config_data.items(): + os.write(fildes, "[%s]\n" % section) + for option, value in values.items(): + os.write(fildes, "%s = %s\n" % (option, value)) + os.close(fildes) + return tmp_file_path + + def run_provider(self): + '''Setup the provider call with proper variables + and call self.get_provider_tags. + ''' + try: + all_results = [] + tmp_file_paths = [] + processes = {} + for account in self.config['accounts']: + env = account['env_vars'] + if account.has_key('provider_config'): + tmp_file_paths.append(MultiEc2.generate_config(account['provider_config'])) + env['EC2_INI_PATH'] = tmp_file_paths[-1] + name = account['name'] + provider = account['provider'] + processes[name] = self.get_provider_tags(provider, env) + + # for each process collect stdout when its available + for name, process in processes.items(): + out, err = process.communicate() + all_results.append({ + "name": name, + "out": out.strip(), + "err": err.strip(), + "code": process.returncode + }) + + finally: + # Clean up the mkstemp file + for tmp_file in tmp_file_paths: + os.unlink(tmp_file) + + return all_results + def get_inventory(self): """Create the subprocess to fetch tags from a provider. Host query: @@ -129,46 +199,61 @@ class MultiEc2(object): Query all of the different accounts for their tags. Once completed store all of their results into one merged updated hash. """ - processes = {} - for account in self.config['accounts']: - env = account['env_vars'] - name = account['name'] - provider = account['provider'] - processes[name] = self.get_provider_tags(provider, env) - - # for each process collect stdout when its available - all_results = [] - for name, process in processes.items(): - out, err = process.communicate() - all_results.append({ - "name": name, - "out": out.strip(), - "err": err.strip(), - "code": process.returncode - }) + provider_results = self.run_provider() # process --host results - if not self.args.host: + # For any 0 result, return it + if self.args.get('host', None): + count = 0 + for results in provider_results: + if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': + self.result = json.loads(results['out']) + count += 1 + if count > 1: + raise RuntimeError("Found > 1 results for --host %s. \ + This is an invalid state." % self.args.get('host', None)) + # process --list results + else: # For any non-zero, raise an error on it - for result in all_results: + for result in provider_results: if result['code'] != 0: raise RuntimeError(result['err']) else: self.all_ec2_results[result['name']] = json.loads(result['out']) + + # Check if user wants extra vars in yaml by + # having hostvars and all_group defined + for acc_config in self.config['accounts']: + self.apply_account_config(acc_config) + + # Build results by merging all dictionaries values = self.all_ec2_results.values() values.insert(0, self.result) for result in values: MultiEc2.merge_destructively(self.result, result) - else: - # For any 0 result, return it - count = 0 - for results in all_results: - if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': - self.result = json.loads(out) - count += 1 - if count > 1: - raise RuntimeError("Found > 1 results for --host %s. \ - This is an invalid state." % self.args.host) + + def apply_account_config(self, acc_config): + ''' Apply account config settings + ''' + if not acc_config.has_key('hostvars') and not acc_config.has_key('all_group'): + return + + results = self.all_ec2_results[acc_config['name']] + # Update each hostvar with the newly desired key: value + for host_property, value in acc_config['hostvars'].items(): + # Verify the account results look sane + # by checking for these keys ('_meta' and 'hostvars' exist) + if results.has_key('_meta') and results['_meta'].has_key('hostvars'): + for data in results['_meta']['hostvars'].values(): + data[str(host_property)] = str(value) + + # Add this group + results["%s_%s" % (host_property, value)] = \ + copy.copy(results[acc_config['all_group']]) + + # store the results back into all_ec2_results + self.all_ec2_results[acc_config['name']] = results + @staticmethod def merge_destructively(input_a, input_b): "merges b into input_a" @@ -182,7 +267,7 @@ class MultiEc2(object): elif isinstance(input_a[key], list) and isinstance(input_b[key], list): for result in input_b[key]: if result not in input_a[key]: - input_a[key].input_append(result) + input_a[key].append(result) # a is a list and not b elif isinstance(input_a[key], list): if input_b[key] not in input_a[key]: @@ -217,14 +302,27 @@ class MultiEc2(object): help='List instances (default: True)') parser.add_argument('--host', action='store', default=False, help='Get all the variables about a specific instance') - self.args = parser.parse_args() + self.args = parser.parse_args().__dict__ def write_to_cache(self): ''' Writes data in JSON format to a file ''' + # if it does not exist, try and create it. + if not os.path.isfile(self.cache_path): + path = os.path.dirname(self.cache_path) + try: + os.makedirs(path) + except OSError as exc: + if exc.errno != errno.EEXIST or not os.path.isdir(path): + raise + json_data = MultiEc2.json_format_dict(self.result, True) with open(self.cache_path, 'w') as cache: - cache.write(json_data) + try: + fcntl.flock(cache, fcntl.LOCK_EX) + cache.write(json_data) + finally: + fcntl.flock(cache, fcntl.LOCK_UN) def get_inventory_from_cache(self): ''' Reads the inventory from the cache file and returns it as a JSON @@ -254,4 +352,7 @@ class MultiEc2(object): if __name__ == "__main__": - print MultiEc2().result_str() + MEC2 = MultiEc2() + MEC2.parse_cli_args() + MEC2.run() + print MEC2.result_str() diff --git a/inventory/multi_ec2.yaml.example b/inventory/multi_ec2.yaml.example index 91e7c7970..99f157b11 100644 --- a/inventory/multi_ec2.yaml.example +++ b/inventory/multi_ec2.yaml.example @@ -1,15 +1,32 @@ # multi ec2 inventory configs +# +cache_location: ~/.ansible/tmp/multi_ec2_inventory.cache + accounts: - name: aws1 provider: aws/hosts/ec2.py + provider_config: + ec2: + regions: all + regions_exclude: us-gov-west-1,cn-north-1 + destination_variable: public_dns_name + route53: False + cache_path: ~/.ansible/tmp + cache_max_age: 300 + vpc_destination_variable: ip_address env_vars: AWS_ACCESS_KEY_ID: XXXXXXXXXXXXXXXXXXXX AWS_SECRET_ACCESS_KEY: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + all_group: ec2 + hostvars: + cloud: aws + account: aws1 - - name: aws2 +- name: aws2 provider: aws/hosts/ec2.py env_vars: AWS_ACCESS_KEY_ID: XXXXXXXXXXXXXXXXXXXX AWS_SECRET_ACCESS_KEY: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + EC2_INI_PATH: /etc/ansible/ec2.ini cache_max_age: 60 diff --git a/inventory/openshift-ansible-inventory.spec b/inventory/openshift-ansible-inventory.spec index 8267e16f6..cd2332549 100644 --- a/inventory/openshift-ansible-inventory.spec +++ b/inventory/openshift-ansible-inventory.spec @@ -1,6 +1,6 @@ Summary: OpenShift Ansible Inventories Name: openshift-ansible-inventory -Version: 0.0.2 +Version: 0.0.7 Release: 1%{?dist} License: ASL 2.0 URL: https://github.com/openshift/openshift-ansible @@ -25,18 +25,39 @@ mkdir -p %{buildroot}/usr/share/ansible/inventory/gce cp -p multi_ec2.py %{buildroot}/usr/share/ansible/inventory cp -p multi_ec2.yaml.example %{buildroot}/etc/ansible/multi_ec2.yaml -cp -p aws/ec2.py aws/ec2.ini %{buildroot}/usr/share/ansible/inventory/aws -cp -p gce/gce.py %{buildroot}/usr/share/ansible/inventory/gce +cp -p aws/hosts/ec2.py %{buildroot}/usr/share/ansible/inventory/aws +cp -p gce/hosts/gce.py %{buildroot}/usr/share/ansible/inventory/gce %files %config(noreplace) /etc/ansible/* %dir /usr/share/ansible/inventory /usr/share/ansible/inventory/multi_ec2.py* /usr/share/ansible/inventory/aws/ec2.py* -%config(noreplace) /usr/share/ansible/inventory/aws/ec2.ini /usr/share/ansible/inventory/gce/gce.py* %changelog +* Fri May 15 2015 Kenny Woodson <kwoodson@redhat.com> 0.0.7-1 +- Making multi_ec2 into a library (kwoodson@redhat.com) + +* Wed May 13 2015 Thomas Wiest <twiest@redhat.com> 0.0.6-1 +- Added support for grouping and a bug fix. (kwoodson@redhat.com) + +* Tue May 12 2015 Thomas Wiest <twiest@redhat.com> 0.0.5-1 +- removed ec2.ini from the openshift-ansible-inventory.spec file so that we're + not dictating what the ec2.ini file should look like. (twiest@redhat.com) +- Added capability to pass in ec2.ini file. (kwoodson@redhat.com) + +* Thu May 07 2015 Thomas Wiest <twiest@redhat.com> 0.0.4-1 +- Fixed a bug due to renaming of variables. (kwoodson@redhat.com) + +* Thu May 07 2015 Thomas Wiest <twiest@redhat.com> 0.0.3-1 +- fixed build problems with openshift-ansible-inventory.spec + (twiest@redhat.com) +- Allow option in multi_ec2 to set cache location. (kwoodson@redhat.com) +- Add ansible_connection=local to localhost in inventory (jdetiber@redhat.com) +- Adding refresh-cache option and cleanup for pylint. Also updated for + aws/hosts/ being added. (kwoodson@redhat.com) + * Thu Mar 26 2015 Thomas Wiest <twiest@redhat.com> 0.0.2-1 - added the ability to have a config file in /etc/openshift_ansible to multi_ec2.py. (twiest@redhat.com) diff --git a/lib/ansible_helper.rb b/lib/ansible_helper.rb deleted file mode 100644 index 395bb51a8..000000000 --- a/lib/ansible_helper.rb +++ /dev/null @@ -1,94 +0,0 @@ -require 'json' -require 'parseconfig' - -module OpenShift - module Ops - class AnsibleHelper - MYDIR = File.expand_path(File.dirname(__FILE__)) - - attr_accessor :inventory, :extra_vars, :verbosity, :pipelining - - def initialize(extra_vars={}, inventory=nil) - @extra_vars = extra_vars - @verbosity = '-vvvv' - @pipelining = true - end - - def all_eof(files) - files.find { |f| !f.eof }.nil? - end - - def run_playbook(playbook) - @inventory = 'inventory/hosts' if @inventory.nil? - - # This is used instead of passing in the json on the cli to avoid quoting problems - tmpfile = Tempfile.open('extra_vars') { |f| f.write(@extra_vars.to_json); f} - - cmds = [] - #cmds << 'set -x' - cmds << %Q[export ANSIBLE_FILTER_PLUGINS="#{Dir.pwd}/filter_plugins"] - - # We need this for launching instances, otherwise conflicting keys and what not kill it - cmds << %q[export ANSIBLE_TRANSPORT="ssh"] - cmds << %q[export ANSIBLE_SSH_ARGS="-o ForwardAgent=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"] - - # We need pipelining off so that we can do sudo to enable the root account - cmds << %Q[export ANSIBLE_SSH_PIPELINING='#{@pipelining.to_s}'] - cmds << %Q[time ansible-playbook -i #{@inventory} #{@verbosity} #{playbook} --extra-vars '@#{tmpfile.path}' ] - cmd = cmds.join(' ; ') - - pid = spawn(cmd, :out => $stdout, :err => $stderr, :close_others => true) - _, state = Process.wait2(pid) - - if 0 != state.exitstatus - raise %Q[Warning failed with exit code: #{state.exitstatus} - -#{cmd} - -extra_vars: #{@extra_vars.to_json} -] - end - ensure - tmpfile.unlink if tmpfile - end - - def merge_extra_vars_file(file) - vars = YAML.load_file(file) - @extra_vars.merge!(vars) - end - - def self.for_gce - ah = AnsibleHelper.new - - # GCE specific configs - gce_ini = "#{MYDIR}/../inventory/gce/gce.ini" - config = ParseConfig.new(gce_ini) - - if config['gce']['gce_project_id'].to_s.empty? - raise %Q['gce_project_id' not set in #{gce_ini}] - end - ah.extra_vars['gce_project_id'] = config['gce']['gce_project_id'] - - if config['gce']['gce_service_account_pem_file_path'].to_s.empty? - raise %Q['gce_service_account_pem_file_path' not set in #{gce_ini}] - end - ah.extra_vars['gce_pem_file'] = config['gce']['gce_service_account_pem_file_path'] - - if config['gce']['gce_service_account_email_address'].to_s.empty? - raise %Q['gce_service_account_email_address' not set in #{gce_ini}] - end - ah.extra_vars['gce_service_account_email'] = config['gce']['gce_service_account_email_address'] - - ah.inventory = 'inventory/gce/gce.py' - return ah - end - - def self.for_aws - ah = AnsibleHelper.new - - ah.inventory = 'inventory/aws/ec2.py' - return ah - end - end - end -end diff --git a/lib/aws_command.rb b/lib/aws_command.rb deleted file mode 100644 index 267513f37..000000000 --- a/lib/aws_command.rb +++ /dev/null @@ -1,148 +0,0 @@ -require 'thor' - -require_relative 'aws_helper' -require_relative 'launch_helper' - -module OpenShift - module Ops - class AwsCommand < Thor - # WARNING: we do not currently support environments with hyphens in the name - SUPPORTED_ENVS = %w(prod stg int ops twiest gshipley kint test jhonce amint tdint lint jdetiber) - - option :type, :required => true, :enum => LaunchHelper.get_aws_host_types, - :desc => 'The host type of the new instances.' - option :env, :required => true, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment of the new instances.' - option :count, :default => 1, :aliases => '-c', :type => :numeric, - :desc => 'The number of instances to create' - option :tag, :type => :array, - :desc => 'The tag(s) to add to the new instances. Allowed characters are letters, numbers, and hyphens.' - desc "launch", "Launches instances." - def launch() - AwsHelper.check_creds() - - # Expand all of the instance names so that we have a complete array - names = [] - options[:count].times { names << "#{options[:env]}-#{options[:type]}-#{SecureRandom.hex(5)}" } - - ah = AnsibleHelper.for_aws() - - # AWS specific configs - ah.extra_vars['oo_new_inst_names'] = names - ah.extra_vars['oo_new_inst_tags'] = options[:tag] - ah.extra_vars['oo_env'] = options[:env] - - # Add a created by tag - ah.extra_vars['oo_new_inst_tags'] = {} if ah.extra_vars['oo_new_inst_tags'].nil? - - ah.extra_vars['oo_new_inst_tags']['created-by'] = ENV['USER'] - ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_env_tag(options[:env])) - ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_host_type_tag(options[:type])) - ah.extra_vars['oo_new_inst_tags'].merge!(AwsHelper.generate_env_host_type_tag(options[:env], options[:type])) - - puts - puts "Creating #{options[:count]} #{options[:type]} instance(s) in AWS..." - - # Make sure we're completely up to date before launching - clear_cache() - ah.run_playbook("playbooks/aws/#{options[:type]}/launch.yml") - ensure - # This is so that if we a config right after a launch, the newly launched instances will be - # in the list. - clear_cache() - end - - desc "clear-cache", 'Clear the inventory cache' - def clear_cache() - print "Clearing inventory cache... " - AwsHelper.clear_inventory_cache() - puts "Done." - end - - option :name, :required => false, :type => :string, - :desc => 'The name of the instance to configure.' - option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment of the new instances.' - option :type, :required => false, :enum => LaunchHelper.get_aws_host_types, - :desc => 'The type of the instances to configure.' - desc "config", 'Configures instances.' - def config() - ah = AnsibleHelper.for_aws() - - abort 'Error: you can\'t specify both --name and --type' unless options[:type].nil? || options[:name].nil? - - abort 'Error: you can\'t specify both --name and --env' unless options[:env].nil? || options[:name].nil? - - host_type = nil - if options[:name] - details = AwsHelper.get_host_details(options[:name]) - ah.extra_vars['oo_host_group_exp'] = details['ec2_public_dns_name'] - ah.extra_vars['oo_env'] = details['ec2_tag_environment'] - host_type = details['ec2_tag_host-type'] - elsif options[:type] && options[:env] - oo_env_host_type_tag = AwsHelper.generate_env_host_type_tag_name(options[:env], options[:type]) - ah.extra_vars['oo_host_group_exp'] = "groups['#{oo_env_host_type_tag}']" - ah.extra_vars['oo_env'] = options[:env] - host_type = options[:type] - else - abort 'Error: you need to specify either --name or (--type and --env)' - end - - puts - puts "Configuring #{options[:type]} instance(s) in AWS..." - - ah.run_playbook("playbooks/aws/#{host_type}/config.yml") - end - - option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment to list.' - desc "list", "Lists instances." - def list() - AwsHelper.check_creds() - hosts = AwsHelper.get_hosts() - - hosts.delete_if { |h| h.env != options[:env] } unless options[:env].nil? - - fmt_str = "%34s %5s %8s %17s %7s" - - puts - puts fmt_str % ['Name','Env', 'State', 'IP Address', 'Created By'] - puts fmt_str % ['----','---', '-----', '----------', '----------'] - hosts.each { |h| puts fmt_str % [h.name, h.env, h.state, h.public_ip, h.created_by ] } - puts - end - - desc "ssh", "Ssh to an instance" - def ssh(*ssh_ops, host) - if host =~ /^([\w\d_.\-]+)@([\w\d\-_.]+)/ - user = $1 - host = $2 - end - - details = AwsHelper.get_host_details(host) - abort "\nError: Instance [#{host}] is not RUNNING\n\n" unless details['ec2_state'] == 'running' - - cmd = "ssh #{ssh_ops.join(' ')}" - - if user.nil? - cmd += " " - else - cmd += " #{user}@" - end - - cmd += "#{details['ec2_ip_address']}" - - exec(cmd) - end - - desc 'types', 'Displays instance types' - def types() - puts - puts "Available Host Types" - puts "--------------------" - LaunchHelper.get_aws_host_types.each { |t| puts " #{t}" } - puts - end - end - end -end diff --git a/lib/aws_helper.rb b/lib/aws_helper.rb deleted file mode 100644 index 4da5d0925..000000000 --- a/lib/aws_helper.rb +++ /dev/null @@ -1,85 +0,0 @@ -require 'fileutils' - -module OpenShift - module Ops - class AwsHelper - MYDIR = File.expand_path(File.dirname(__FILE__)) - - def self.get_list() - cmd = "#{MYDIR}/../inventory/aws/ec2.py --list" - hosts = %x[#{cmd} 2>&1] - - raise "Error: failed to list hosts\n#{hosts}" unless $?.exitstatus == 0 - return JSON.parse(hosts) - end - - def self.get_hosts() - hosts = get_list() - - retval = [] - hosts['_meta']['hostvars'].each do |host, info| - retval << OpenStruct.new({ - :name => info['ec2_tag_Name'] || 'UNSET', - :env => info['ec2_tag_environment'] || 'UNSET', - :public_ip => info['ec2_ip_address'], - :public_dns => info['ec2_public_dns_name'], - :state => info['ec2_state'], - :created_by => info['ec2_tag_created-by'] - }) - end - - retval.sort_by! { |h| [h.env, h.state, h.name] } - - return retval - end - - def self.get_host_details(host) - hosts = get_list() - dns_names = hosts["tag_Name_#{host}"] - - raise "Host not found [#{host}]" if dns_names.nil? - raise "Multiple entries found for [#{host}]" if dns_names.size > 1 - - return hosts['_meta']['hostvars'][dns_names.first] - end - - def self.check_creds() - raise "AWS_ACCESS_KEY_ID environment variable must be set" if ENV['AWS_ACCESS_KEY_ID'].nil? - raise "AWS_SECRET_ACCESS_KEY environment variable must be set" if ENV['AWS_SECRET_ACCESS_KEY'].nil? - end - - def self.clear_inventory_cache() - path = "#{ENV['HOME']}/.ansible/tmp" - cache_files = ["#{path}/ansible-ec2.cache", "#{path}/ansible-ec2.index"] - FileUtils.rm_f(cache_files) - end - - def self.generate_env_tag(env) - return { "environment" => env } - end - - def self.generate_env_tag_name(env) - h = generate_env_tag(env) - return "tag_#{h.keys.first}_#{h.values.first}" - end - - def self.generate_host_type_tag(host_type) - return { "host-type" => host_type } - end - - def self.generate_host_type_tag_name(host_type) - h = generate_host_type_tag(host_type) - return "tag_#{h.keys.first}_#{h.values.first}" - end - - def self.generate_env_host_type_tag(env, host_type) - return { "env-host-type" => "#{env}-#{host_type}" } - end - - def self.generate_env_host_type_tag_name(env, host_type) - h = generate_env_host_type_tag(env, host_type) - return "tag_#{h.keys.first}_#{h.values.first}" - end - end - end -end diff --git a/lib/gce_command.rb b/lib/gce_command.rb deleted file mode 100644 index 214cc1c05..000000000 --- a/lib/gce_command.rb +++ /dev/null @@ -1,228 +0,0 @@ -require 'thor' -require 'securerandom' -require 'fileutils' - -require_relative 'gce_helper' -require_relative 'launch_helper' -require_relative 'ansible_helper' - -module OpenShift - module Ops - class GceCommand < Thor - # WARNING: we do not currently support environments with hyphens in the name - SUPPORTED_ENVS = %w(prod stg int twiest gshipley kint test jhonce amint tdint lint jdetiber) - - option :type, :required => true, :enum => LaunchHelper.get_gce_host_types, - :desc => 'The host type of the new instances.' - option :env, :required => true, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment of the new instances.' - option :count, :default => 1, :aliases => '-c', :type => :numeric, - :desc => 'The number of instances to create' - option :tag, :type => :array, - :desc => 'The tag(s) to add to the new instances. Allowed characters are letters, numbers, and hyphens.' - desc "launch", "Launches instances." - def launch() - # Expand all of the instance names so that we have a complete array - names = [] - options[:count].times { names << "#{options[:env]}-#{options[:type]}-#{SecureRandom.hex(5)}" } - - ah = AnsibleHelper.for_gce() - - # GCE specific configs - ah.extra_vars['oo_new_inst_names'] = names - ah.extra_vars['oo_new_inst_tags'] = options[:tag] - ah.extra_vars['oo_env'] = options[:env] - - # Add a created by tag - ah.extra_vars['oo_new_inst_tags'] = [] if ah.extra_vars['oo_new_inst_tags'].nil? - - ah.extra_vars['oo_new_inst_tags'] << "created-by-#{ENV['USER']}" - ah.extra_vars['oo_new_inst_tags'] << GceHelper.generate_env_tag(options[:env]) - ah.extra_vars['oo_new_inst_tags'] << GceHelper.generate_host_type_tag(options[:type]) - ah.extra_vars['oo_new_inst_tags'] << GceHelper.generate_env_host_type_tag(options[:env], options[:type]) - - puts - puts "Creating #{options[:count]} #{options[:type]} instance(s) in GCE..." - - ah.run_playbook("playbooks/gce/#{options[:type]}/launch.yml") - end - - - option :name, :required => false, :type => :string, - :desc => 'The name of the instance to configure.' - option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment of the new instances.' - option :type, :required => false, :enum => LaunchHelper.get_gce_host_types, - :desc => 'The type of the instances to configure.' - desc "config", 'Configures instances.' - def config() - ah = AnsibleHelper.for_gce() - - abort 'Error: you can\'t specify both --name and --type' unless options[:type].nil? || options[:name].nil? - - abort 'Error: you can\'t specify both --name and --env' unless options[:env].nil? || options[:name].nil? - - host_type = nil - if options[:name] - details = GceHelper.get_host_details(options[:name]) - ah.extra_vars['oo_host_group_exp'] = options[:name] - ah.extra_vars['oo_env'] = details['env'] - host_type = details['host-type'] - elsif options[:type] && options[:env] - oo_env_host_type_tag = GceHelper.generate_env_host_type_tag_name(options[:env], options[:type]) - ah.extra_vars['oo_host_group_exp'] = "groups['#{oo_env_host_type_tag}']" - ah.extra_vars['oo_env'] = options[:env] - host_type = options[:type] - else - abort 'Error: you need to specify either --name or (--type and --env)' - end - - puts - puts "Configuring #{options[:type]} instance(s) in GCE..." - - ah.run_playbook("playbooks/gce/#{host_type}/config.yml") - end - - option :name, :required => false, :type => :string, - :desc => 'The name of the instance to terminate.' - option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment of the new instances.' - option :type, :required => false, :enum => LaunchHelper.get_gce_host_types, - :desc => 'The type of the instances to configure.' - option :confirm, :required => false, :type => :boolean, - :desc => 'Terminate without interactive confirmation' - desc "terminate", 'Terminate instances' - def terminate() - ah = AnsibleHelper.for_gce() - - abort 'Error: you can\'t specify both --name and --type' unless options[:type].nil? || options[:name].nil? - - abort 'Error: you can\'t specify both --name and --env' unless options[:env].nil? || options[:name].nil? - - host_type = nil - if options[:name] - details = GceHelper.get_host_details(options[:name]) - ah.extra_vars['oo_host_group_exp'] = options[:name] - ah.extra_vars['oo_env'] = details['env'] - host_type = details['host-type'] - elsif options[:type] && options[:env] - oo_env_host_type_tag = GceHelper.generate_env_host_type_tag_name(options[:env], options[:type]) - ah.extra_vars['oo_host_group_exp'] = "groups['#{oo_env_host_type_tag}']" - ah.extra_vars['oo_env'] = options[:env] - host_type = options[:type] - else - abort 'Error: you need to specify either --name or (--type and --env)' - end - - puts - puts "Terminating #{options[:type]} instance(s) in GCE..." - - ah.run_playbook("playbooks/gce/#{host_type}/terminate.yml") - end - - option :env, :required => false, :aliases => '-e', :enum => SUPPORTED_ENVS, - :desc => 'The environment to list.' - desc "list", "Lists instances." - def list() - hosts = GceHelper.get_hosts() - - hosts.delete_if { |h| h.env != options[:env] } unless options[:env].nil? - - fmt_str = "%34s %5s %8s %17s %7s" - - puts - puts fmt_str % ['Name','Env', 'State', 'IP Address', 'Created By'] - puts fmt_str % ['----','---', '-----', '----------', '----------'] - hosts.each { |h| puts fmt_str % [h.name, h.env, h.state, h.public_ip, h.created_by ] } - puts - end - - option :file, :required => true, :type => :string, - :desc => 'The name of the file to copy.' - option :dest, :required => false, :type => :string, - :desc => 'A relative path where files are written to.' - desc "scp_from", "scp files from an instance" - def scp_from(*ssh_ops, host) - if host =~ /^([\w\d_.\-]+)@([\w\d\-_.]+)$/ - user = $1 - host = $2 - end - - path_to_file = options['file'] - dest = options['dest'] - - details = GceHelper.get_host_details(host) - abort "\nError: Instance [#{host}] is not RUNNING\n\n" unless details['gce_status'] == 'RUNNING' - - cmd = "scp #{ssh_ops.join(' ')}" - - if user.nil? - cmd += " " - else - cmd += " #{user}@" - end - - if dest.nil? - download = File.join(Dir.pwd, 'download') - FileUtils.mkdir_p(download) unless File.exists?(download) - cmd += "#{details['gce_public_ip']}:#{path_to_file} download/" - else - cmd += "#{details['gce_public_ip']}:#{path_to_file} #{File.expand_path(dest)}" - end - - exec(cmd) - end - - desc "ssh", "Ssh to an instance" - def ssh(*ssh_ops, host) - if host =~ /^([\w\d_.\-]+)@([\w\d\-_.]+)/ - user = $1 - host = $2 - end - - details = GceHelper.get_host_details(host) - abort "\nError: Instance [#{host}] is not RUNNING\n\n" unless details['gce_status'] == 'RUNNING' - - cmd = "ssh #{ssh_ops.join(' ')}" - - if user.nil? - cmd += " " - else - cmd += " #{user}@" - end - - cmd += "#{details['gce_public_ip']}" - - exec(cmd) - end - - option :name, :required => true, :aliases => '-n', :type => :string, - :desc => 'The name of the instance.' - desc 'details', 'Displays details about an instance.' - def details() - name = options[:name] - - details = GceHelper.get_host_details(name) - - key_size = details.keys.max_by { |k| k.size }.size - - header = "Details for #{name}" - puts - puts header - header.size.times { print '-' } - puts - details.each { |k,v| printf("%#{key_size + 2}s: %s\n", k, v) } - puts - end - - desc 'types', 'Displays instance types' - def types() - puts - puts "Available Host Types" - puts "--------------------" - LaunchHelper.get_gce_host_types.each { |t| puts " #{t}" } - puts - end - end - end -end diff --git a/lib/gce_helper.rb b/lib/gce_helper.rb deleted file mode 100644 index 19fa00020..000000000 --- a/lib/gce_helper.rb +++ /dev/null @@ -1,94 +0,0 @@ -require 'ostruct' - -module OpenShift - module Ops - class GceHelper - MYDIR = File.expand_path(File.dirname(__FILE__)) - - def self.get_list() - cmd = "#{MYDIR}/../inventory/gce/gce.py --list" - hosts = %x[#{cmd} 2>&1] - - raise "Error: failed to list hosts\n#{hosts}" unless $?.exitstatus == 0 - - return JSON.parse(hosts) - end - - def self.get_tag(tags, selector) - tags.each do |tag| - return $1 if tag =~ selector - end - - return nil - end - - def self.get_hosts() - hosts = get_list() - - retval = [] - hosts['_meta']['hostvars'].each do |host, info| - retval << OpenStruct.new({ - :name => info['gce_name'], - :env => get_tag(info['gce_tags'], /^env-(\w+)$/) || 'UNSET', - :public_ip => info['gce_public_ip'], - :state => info['gce_status'], - :created_by => get_tag(info['gce_tags'], /^created-by-(\w+)$/) || 'UNSET', - }) - end - - retval.sort_by! { |h| [h.env, h.state, h.name] } - - return retval - - end - - def self.get_host_details(host) - cmd = "#{MYDIR}/../inventory/gce/gce.py --host #{host}" - details = %x[#{cmd} 2>&1] - - raise "Error: failed to get host details\n#{details}" unless $?.exitstatus == 0 - - retval = JSON.parse(details) - - raise "Error: host not found [#{host}]" if retval.empty? - - # Convert OpenShift specific tags to entries - retval['gce_tags'].each do |tag| - if tag =~ /\Ahost-type-([\w\d-]+)\z/ - retval['host-type'] = $1 - end - - if tag =~ /\Aenv-([\w\d]+)\z/ - retval['env'] = $1 - end - end - - return retval - end - - def self.generate_env_tag(env) - return "env-#{env}" - end - - def self.generate_env_tag_name(env) - return "tag_#{generate_env_tag(env)}" - end - - def self.generate_host_type_tag(host_type) - return "host-type-#{host_type}" - end - - def self.generate_host_type_tag_name(host_type) - return "tag_#{generate_host_type_tag(host_type)}" - end - - def self.generate_env_host_type_tag(env, host_type) - return "env-host-type-#{env}-#{host_type}" - end - - def self.generate_env_host_type_tag_name(env, host_type) - return "tag_#{generate_env_host_type_tag(env, host_type)}" - end - end - end -end diff --git a/lib/launch_helper.rb b/lib/launch_helper.rb deleted file mode 100644 index 0fe5ea6dc..000000000 --- a/lib/launch_helper.rb +++ /dev/null @@ -1,30 +0,0 @@ -module OpenShift - module Ops - class LaunchHelper - MYDIR = File.expand_path(File.dirname(__FILE__)) - - def self.expand_name(name) - return [name] unless name =~ /^([a-zA-Z0-9\-]+)\{(\d+)-(\d+)\}$/ - - # Regex matched, so grab the values - start_num = $2 - end_num = $3 - - retval = [] - start_num.upto(end_num) do |i| - retval << "#{$1}#{i}" - end - - return retval - end - - def self.get_gce_host_types() - return Dir.glob("#{MYDIR}/../playbooks/gce/*").map { |d| File.basename(d) } - end - - def self.get_aws_host_types() - return Dir.glob("#{MYDIR}/../playbooks/aws/*").map { |d| File.basename(d) } - end - end - end -end diff --git a/playbooks/aws/ansible-tower/launch.yml b/playbooks/aws/ansible-tower/launch.yml index 56235bc8a..c23bda3a0 100644 --- a/playbooks/aws/ansible-tower/launch.yml +++ b/playbooks/aws/ansible-tower/launch.yml @@ -6,7 +6,7 @@ vars: inst_region: us-east-1 - rhel7_ami: ami-906240f8 + rhel7_ami: ami-78756d10 user_data_file: user_data.txt vars_files: diff --git a/playbooks/aws/openshift-cluster/launch.yml b/playbooks/aws/openshift-cluster/launch.yml index 3eb5496e4..33e1ec25d 100644 --- a/playbooks/aws/openshift-cluster/launch.yml +++ b/playbooks/aws/openshift-cluster/launch.yml @@ -25,6 +25,14 @@ cluster: "{{ cluster_id }}" type: "{{ k8s_type }}" + - set_fact: + a_master: "{{ master_names[0] }}" + - add_host: name={{ a_master }} groups=service_master + - include: update.yml +- include: ../../common/openshift-cluster/create_services.yml + vars: + g_svc_master: "{{ service_master }}" + - include: list.yml diff --git a/playbooks/aws/openshift-cluster/vars.online.int.yml b/playbooks/aws/openshift-cluster/vars.online.int.yml index 12f79a9c1..e115615d5 100644 --- a/playbooks/aws/openshift-cluster/vars.online.int.yml +++ b/playbooks/aws/openshift-cluster/vars.online.int.yml @@ -1,5 +1,5 @@ --- -ec2_image: ami-906240f8 +ec2_image: ami-78756d10 ec2_image_name: libra-ops-rhel7* ec2_region: us-east-1 ec2_keypair: mmcgrath_libra diff --git a/playbooks/aws/openshift-cluster/vars.online.prod.yml b/playbooks/aws/openshift-cluster/vars.online.prod.yml index 12f79a9c1..e115615d5 100644 --- a/playbooks/aws/openshift-cluster/vars.online.prod.yml +++ b/playbooks/aws/openshift-cluster/vars.online.prod.yml @@ -1,5 +1,5 @@ --- -ec2_image: ami-906240f8 +ec2_image: ami-78756d10 ec2_image_name: libra-ops-rhel7* ec2_region: us-east-1 ec2_keypair: mmcgrath_libra diff --git a/playbooks/aws/openshift-cluster/vars.online.stage.yml b/playbooks/aws/openshift-cluster/vars.online.stage.yml index 12f79a9c1..e115615d5 100644 --- a/playbooks/aws/openshift-cluster/vars.online.stage.yml +++ b/playbooks/aws/openshift-cluster/vars.online.stage.yml @@ -1,5 +1,5 @@ --- -ec2_image: ami-906240f8 +ec2_image: ami-78756d10 ec2_image_name: libra-ops-rhel7* ec2_region: us-east-1 ec2_keypair: mmcgrath_libra diff --git a/playbooks/aws/openshift-master/launch.yml b/playbooks/aws/openshift-master/launch.yml index 6b3751682..51a0258f0 100644 --- a/playbooks/aws/openshift-master/launch.yml +++ b/playbooks/aws/openshift-master/launch.yml @@ -4,10 +4,10 @@ connection: local gather_facts: no -# TODO: modify atomic_ami based on deployment_type +# TODO: modify g_ami based on deployment_type vars: inst_region: us-east-1 - atomic_ami: ami-86781fee + g_ami: ami-86781fee user_data_file: user_data.txt tasks: @@ -18,13 +18,13 @@ keypair: libra group: ['public'] instance_type: m3.large - image: "{{ atomic_ami }}" + image: "{{ g_ami }}" count: "{{ oo_new_inst_names | oo_len }}" user_data: "{{ lookup('file', user_data_file) }}" wait: yes register: ec2 - - name: Add new instances public IPs to the atomic proxy host group + - name: Add new instances public IPs to the host group add_host: "hostname={{ item.public_ip }} groupname=new_ec2_instances" with_items: ec2.instances diff --git a/playbooks/aws/openshift-node/launch.yml b/playbooks/aws/openshift-node/launch.yml index 36aee14ff..d6024a020 100644 --- a/playbooks/aws/openshift-node/launch.yml +++ b/playbooks/aws/openshift-node/launch.yml @@ -4,10 +4,10 @@ connection: local gather_facts: no -# TODO: modify atomic_ami based on deployment_type +# TODO: modify g_ami based on deployment_type vars: inst_region: us-east-1 - atomic_ami: ami-86781fee + g_ami: ami-86781fee user_data_file: user_data.txt tasks: @@ -18,13 +18,13 @@ keypair: libra group: ['public'] instance_type: m3.large - image: "{{ atomic_ami }}" + image: "{{ g_ami }}" count: "{{ oo_new_inst_names | oo_len }}" user_data: "{{ lookup('file', user_data_file) }}" wait: yes register: ec2 - - name: Add new instances public IPs to the atomic proxy host group + - name: Add new instances public IPs to the host group add_host: hostname: "{{ item.public_ip }}" groupname: new_ec2_instances" diff --git a/playbooks/aws/os2-atomic-proxy/config.yml b/playbooks/aws/os2-atomic-proxy/config.yml deleted file mode 100644 index 7d384a665..000000000 --- a/playbooks/aws/os2-atomic-proxy/config.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: "populate oo_hosts_to_config host group if needed" - hosts: localhost - gather_facts: no - tasks: - - name: Evaluate oo_host_group_exp if it's set - add_host: "name={{ item }} groups=oo_hosts_to_config" - with_items: "{{ oo_host_group_exp | default(['']) }}" - when: oo_host_group_exp is defined - -- name: "Configure instances" - hosts: oo_hosts_to_config - connection: ssh - user: root - vars_files: - - vars.yml - - "vars.{{ oo_env }}.yml" - roles: - - atomic_base - - atomic_proxy diff --git a/playbooks/aws/os2-atomic-proxy/filter_plugins b/playbooks/aws/os2-atomic-proxy/filter_plugins deleted file mode 120000 index 99a95e4ca..000000000 --- a/playbooks/aws/os2-atomic-proxy/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/aws/os2-atomic-proxy/launch.yml b/playbooks/aws/os2-atomic-proxy/launch.yml deleted file mode 100644 index fd6b0f39a..000000000 --- a/playbooks/aws/os2-atomic-proxy/launch.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -- name: Launch instance(s) - hosts: localhost - connection: local - gather_facts: no - - vars: - inst_region: us-east-1 - atomic_ami: ami-8e239fe6 - user_data_file: user_data.txt - oo_vpc_subnet_id: # Purposely left blank, these are here to be overridden in env vars_files - oo_assign_public_ip: # Purposely left blank, these are here to be overridden in env vars_files - - vars_files: - - vars.yml - - "vars.{{ oo_env }}.yml" - - tasks: - - name: Launch instances in VPC - ec2: - state: present - region: "{{ inst_region }}" - keypair: mmcgrath_libra - group_id: "{{ oo_security_group_ids }}" - instance_type: m3.large - image: "{{ atomic_ami }}" - count: "{{ oo_new_inst_names | oo_len }}" - user_data: "{{ lookup('file', user_data_file) }}" - wait: yes - assign_public_ip: "{{ oo_assign_public_ip }}" - vpc_subnet_id: "{{ oo_vpc_subnet_id }}" - when: oo_vpc_subnet_id - register: ec2_vpc - - - set_fact: - ec2: "{{ ec2_vpc }}" - when: oo_vpc_subnet_id - - - name: Launch instances in Classic - ec2: - state: present - region: "{{ inst_region }}" - keypair: mmcgrath_libra - group: ['Libra', '{{ oo_env }}', '{{ oo_env }}_proxy', '{{ oo_env }}_proxy_atomic'] - instance_type: m3.large - image: "{{ atomic_ami }}" - count: "{{ oo_new_inst_names | oo_len }}" - user_data: "{{ lookup('file', user_data_file) }}" - wait: yes - when: not oo_vpc_subnet_id - register: ec2_classic - - - set_fact: - ec2: "{{ ec2_classic }}" - when: not oo_vpc_subnet_id - - - name: Add new instances public IPs to the atomic proxy host group - add_host: "hostname={{ item.public_ip }} groupname=new_ec2_instances" - with_items: ec2.instances - - - name: Add Name and environment tags to instances - ec2_tag: "resource={{ item.1.id }} region={{ inst_region }} state=present" - with_together: - - oo_new_inst_names - - ec2.instances - args: - tags: - Name: "{{ item.0 }}" - - - name: Add other tags to instances - ec2_tag: "resource={{ item.id }} region={{ inst_region }} state=present" - with_items: ec2.instances - args: - tags: "{{ oo_new_inst_tags }}" - - - name: Add new instances public IPs to oo_hosts_to_config - add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.public_ip }} groupname=oo_hosts_to_config" - with_together: - - oo_new_inst_names - - ec2.instances - - - debug: var=ec2 - - - name: Wait for ssh - wait_for: "port=22 host={{ item.public_ip }}" - with_items: ec2.instances - - - name: Wait for root user setup - command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null root@{{ item.public_ip }} echo root user is setup" - register: result - until: result.rc == 0 - retries: 20 - delay: 10 - with_items: ec2.instances - -# Apply the configs, seprate so that just the configs can be run by themselves -- include: config.yml diff --git a/playbooks/aws/os2-atomic-proxy/roles b/playbooks/aws/os2-atomic-proxy/roles deleted file mode 120000 index 20c4c58cf..000000000 --- a/playbooks/aws/os2-atomic-proxy/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles
\ No newline at end of file diff --git a/playbooks/aws/os2-atomic-proxy/user_data.txt b/playbooks/aws/os2-atomic-proxy/user_data.txt deleted file mode 100644 index 643d17c32..000000000 --- a/playbooks/aws/os2-atomic-proxy/user_data.txt +++ /dev/null @@ -1,6 +0,0 @@ -#cloud-config -disable_root: 0 - -system_info: - default_user: - name: root diff --git a/playbooks/aws/os2-atomic-proxy/vars.int.yml b/playbooks/aws/os2-atomic-proxy/vars.int.yml deleted file mode 100644 index 00157cd89..000000000 --- a/playbooks/aws/os2-atomic-proxy/vars.int.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -oo_env_long: integration -oo_zabbix_hostgroups: ['INT Environment'] diff --git a/playbooks/aws/os2-atomic-proxy/vars.prod.yml b/playbooks/aws/os2-atomic-proxy/vars.prod.yml deleted file mode 100644 index 641afc626..000000000 --- a/playbooks/aws/os2-atomic-proxy/vars.prod.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -oo_env_long: production -oo_zabbix_hostgroups: ['PROD Environment'] diff --git a/playbooks/aws/os2-atomic-proxy/vars.stg.yml b/playbooks/aws/os2-atomic-proxy/vars.stg.yml deleted file mode 100644 index 1cecfc9b2..000000000 --- a/playbooks/aws/os2-atomic-proxy/vars.stg.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -oo_env_long: staging -oo_zabbix_hostgroups: ['STG Environment'] -oo_vpc_subnet_id: subnet-700bdd07 -oo_assign_public_ip: yes -oo_security_group_ids: - - sg-02c2f267 # Libra (vpc) - - sg-f0bfbe95 # stg (vpc) - - sg-a3bfbec6 # stg_proxy (vpc) - - sg-d4bfbeb1 # stg_proxy_atomic (vpc) diff --git a/playbooks/byo/config.yml b/playbooks/byo/config.yml index dce49d32f..e059514db 100644 --- a/playbooks/byo/config.yml +++ b/playbooks/byo/config.yml @@ -1,6 +1,8 @@ --- - name: Run the openshift-master config playbook include: openshift-master/config.yml + when: groups.masters is defined and groups.masters - name: Run the openshift-node config playbook include: openshift-node/config.yml + when: groups.nodes is defined and groups.nodes and groups.masters is defined and groups.masters diff --git a/playbooks/common/openshift-cluster/create_services.yml b/playbooks/common/openshift-cluster/create_services.yml new file mode 100644 index 000000000..e70709d19 --- /dev/null +++ b/playbooks/common/openshift-cluster/create_services.yml @@ -0,0 +1,8 @@ +--- +- name: Deploy OpenShift Services + hosts: "{{ g_svc_master }}" + connection: ssh + gather_facts: yes + roles: + - openshift_registry + - openshift_router diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 05822d118..4df64e95f 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -6,6 +6,7 @@ roles: - openshift_master - { role: openshift_sdn_master, when: openshift.common.use_openshift_sdn | bool } + - { role: fluentd_master, when openshift.common.use_fluentd | bool } tasks: - name: Create group for deployment type group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }} diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml index 433cfeb87..70711e39b 100644 --- a/playbooks/common/openshift-node/config.yml +++ b/playbooks/common/openshift-node/config.yml @@ -15,6 +15,7 @@ local_facts: hostname: "{{ openshift_hostname | default(None) }}" public_hostname: "{{ openshift_public_hostname | default(None) }}" + deployment_type: "{{ openshift_deployment_type }}" - role: node local_facts: external_id: "{{ openshift_node_external_id | default(None) }}" @@ -23,7 +24,6 @@ pod_cidr: "{{ openshift_node_pod_cidr | default(None) }}" labels: "{{ openshift_node_labels | default(None) }}" annotations: "{{ openshift_node_annotations | default(None) }}" - deployment_type: "{{ openshift_deployment_type }}" - name: Create temp directory for syncing certs @@ -68,7 +68,6 @@ fetch: src: "{{ sync_tmpdir }}/{{ item.openshift.common.hostname }}.tgz" dest: "{{ sync_tmpdir }}/" - flat: yes fail_on_missing: yes validate_checksum: yes with_items: openshift_nodes @@ -79,7 +78,7 @@ hosts: oo_nodes_to_config gather_facts: no vars: - sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}" + sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}/{{ groups['oo_first_master'][0] }}/{{ hostvars.localhost.mktemp.stdout }}" openshift_sdn_master_url: "https://{{ hostvars[groups['oo_first_master'][0]].openshift.common.hostname }}:4001" pre_tasks: - name: Ensure certificate directory exists @@ -97,6 +96,7 @@ roles: - openshift_node - { role: openshift_sdn_node, when: openshift.common.use_openshift_sdn | bool } + - { role: fluentd_node, when: openshift.common.use_fluentd | bool } tasks: - name: Create group for deployment type group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }} diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml index 771f51e91..35737f03d 100644 --- a/playbooks/gce/openshift-cluster/launch.yml +++ b/playbooks/gce/openshift-cluster/launch.yml @@ -23,6 +23,22 @@ cluster: "{{ cluster_id }}" type: "{{ k8s_type }}" + - set_fact: + a_master: "{{ master_names[0] }}" + - add_host: name={{ a_master }} groups=service_master + - include: update.yml +- name: Deploy OpenShift Services + hosts: service_master + connection: ssh + gather_facts: yes + roles: + - openshift_registry + - openshift_router + +- include: ../../common/openshift-cluster/create_services.yml + vars: + g_svc_master: "{{ service_master }}" + - include: list.yml diff --git a/playbooks/gce/openshift-cluster/list.yml b/playbooks/gce/openshift-cluster/list.yml index 962381306..5ba0f5a48 100644 --- a/playbooks/gce/openshift-cluster/list.yml +++ b/playbooks/gce/openshift-cluster/list.yml @@ -16,7 +16,7 @@ ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}" with_items: groups[scratch_group] | default([]) | difference(['localhost']) | difference(groups.status_terminated) -- name: List Hosts +- name: List instance(s) hosts: oo_list_hosts gather_facts: no tasks: diff --git a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml index 359d0b2f3..8bf1e84ee 100644 --- a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml +++ b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml @@ -58,23 +58,17 @@ uri: '{{ libvirt_uri }}' with_items: instances -- name: Collect MAC addresses of the VMs - shell: 'virsh -c {{ libvirt_uri }} dumpxml {{ item }} | xmllint --xpath "string(//domain/devices/interface/mac/@address)" -' - register: scratch_mac - with_items: instances - - name: Wait for the VMs to get an IP - command: "egrep -c '{{ scratch_mac.results | oo_collect('stdout') | join('|') }}' /proc/net/arp" - ignore_errors: yes + shell: 'virsh net-dhcp-leases openshift-ansible | egrep -c ''{{ instances | join("|") }}''' register: nb_allocated_ips until: nb_allocated_ips.stdout == '{{ instances | length }}' retries: 30 delay: 1 - name: Collect IP addresses of the VMs - shell: "awk '/{{ item.stdout }}/ {print $1}' /proc/net/arp" + shell: 'virsh net-dhcp-leases openshift-ansible | awk ''$6 == "{{ item }}" {gsub(/\/.*/, "", $5); print $5}''' register: scratch_ip - with_items: scratch_mac.results + with_items: instances - set_fact: ips: "{{ scratch_ip.results | oo_collect('stdout') }}" diff --git a/rel-eng/packages/openshift-ansible-bin b/rel-eng/packages/openshift-ansible-bin index 8a9624397..de9bb5157 100644 --- a/rel-eng/packages/openshift-ansible-bin +++ b/rel-eng/packages/openshift-ansible-bin @@ -1 +1 @@ -0.0.12-1 bin/ +0.0.17-1 bin/ diff --git a/rel-eng/packages/openshift-ansible-inventory b/rel-eng/packages/openshift-ansible-inventory index cf3ac87ed..df529d9fd 100644 --- a/rel-eng/packages/openshift-ansible-inventory +++ b/rel-eng/packages/openshift-ansible-inventory @@ -1 +1 @@ -0.0.2-1 inventory/ +0.0.7-1 inventory/ diff --git a/roles/ansible/tasks/config.yml b/roles/ansible/tasks/config.yml new file mode 100644 index 000000000..5e361429b --- /dev/null +++ b/roles/ansible/tasks/config.yml @@ -0,0 +1,8 @@ +--- +- name: modify ansible.cfg + lineinfile: + dest: /etc/ansible/ansible.cfg + backrefs: yes + regexp: "^#?({{ item.option }})( *)=" + line: '\1\2= {{ item.value }}' + with_items: cfg_options diff --git a/roles/ansible/tasks/main.yaml b/roles/ansible/tasks/main.yml index 67a04b919..5d20a3b35 100644 --- a/roles/ansible/tasks/main.yaml +++ b/roles/ansible/tasks/main.yml @@ -5,3 +5,7 @@ yum: pkg: ansible state: installed + +- include: config.yml + vars: + cfg_options: "{{ ans_config }}" diff --git a/roles/atomic_base/README.md b/roles/atomic_base/README.md deleted file mode 100644 index 8fe3faf7d..000000000 --- a/roles/atomic_base/README.md +++ /dev/null @@ -1,56 +0,0 @@ -Role Name -======== - -The purpose of this role is to do common configurations for all RHEL atomic hosts. - - -Requirements ------------- - -None - - -Role Variables --------------- - -None - - -Dependencies ------------- - -None - - -Example Playbook -------------------------- - -From a group playbook: - - hosts: servers - roles: - - ../../roles/atomic_base - - -License -------- - -Copyright 2012-2014 Red Hat, Inc., All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -Author Information ------------------- - -Thomas Wiest <twiest@redhat.com> diff --git a/roles/atomic_base/files/bash/bashrc b/roles/atomic_base/files/bash/bashrc deleted file mode 100644 index 446f18f22..000000000 --- a/roles/atomic_base/files/bash/bashrc +++ /dev/null @@ -1,12 +0,0 @@ -# .bashrc - -# User specific aliases and functions - -alias rm='rm -i' -alias cp='cp -i' -alias mv='mv -i' - -# Source global definitions -if [ -f /etc/bashrc ]; then - . /etc/bashrc -fi diff --git a/roles/atomic_base/files/ostree/repo_config b/roles/atomic_base/files/ostree/repo_config deleted file mode 100644 index 7038158f9..000000000 --- a/roles/atomic_base/files/ostree/repo_config +++ /dev/null @@ -1,10 +0,0 @@ -[core] -repo_version=1 -mode=bare - -[remote "rh-atomic-controller"] -url=https://mirror.openshift.com/libra/ostree/rhel-7-atomic-host -branches=rh-atomic-controller/el7/x86_64/buildmaster/controller/docker; -tls-client-cert-path=/var/lib/yum/client-cert.pem -tls-client-key-path=/var/lib/yum/client-key.pem -gpg-verify=false diff --git a/roles/atomic_base/files/system/90-nofile.conf b/roles/atomic_base/files/system/90-nofile.conf deleted file mode 100644 index 8537a4c5f..000000000 --- a/roles/atomic_base/files/system/90-nofile.conf +++ /dev/null @@ -1,7 +0,0 @@ -# PAM process file descriptor limits -# see limits.conf(5) for details. -#Each line describes a limit for a user in the form: -# -#<domain> <type> <item> <value> -* hard nofile 16384 -root soft nofile 16384 diff --git a/roles/atomic_base/meta/main.yml b/roles/atomic_base/meta/main.yml deleted file mode 100644 index 9578ab809..000000000 --- a/roles/atomic_base/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -galaxy_info: - author: Thomas Wiest - description: Common base RHEL atomic configurations - company: Red Hat - # Some suggested licenses: - # - BSD (default) - # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: Apache - min_ansible_version: 1.2 - platforms: - - name: EL - versions: - - 7 -dependencies: [] diff --git a/roles/atomic_base/tasks/bash.yml b/roles/atomic_base/tasks/bash.yml deleted file mode 100644 index 547ae83c3..000000000 --- a/roles/atomic_base/tasks/bash.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Copy .bashrc - copy: src=bash/bashrc dest=/root/.bashrc owner=root group=root mode=0644 - -- name: Link to .profile to .bashrc - file: src=/root/.bashrc dest=/root/.profile owner=root group=root state=link - -- name: "Setup Timezone [{{ oo_timezone }}]" - file: - src: "/usr/share/zoneinfo/{{ oo_timezone }}" - dest: /etc/localtime - owner: root - group: root - state: link diff --git a/roles/atomic_base/tasks/cloud_user.yml b/roles/atomic_base/tasks/cloud_user.yml deleted file mode 100644 index e7347fc3d..000000000 --- a/roles/atomic_base/tasks/cloud_user.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Remove cloud-user account - user: name=cloud-user state=absent remove=yes force=yes - -- name: Remove cloud-user sudo - file: path=/etc/sudoers.d/90-cloud-init-users state=absent diff --git a/roles/atomic_base/tasks/main.yml b/roles/atomic_base/tasks/main.yml deleted file mode 100644 index 5d8e8571a..000000000 --- a/roles/atomic_base/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: system.yml -- include: bash.yml -- include: ostree.yml diff --git a/roles/atomic_base/tasks/ostree.yml b/roles/atomic_base/tasks/ostree.yml deleted file mode 100644 index aacaa5efd..000000000 --- a/roles/atomic_base/tasks/ostree.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: Copy ostree repo config - copy: - src: ostree/repo_config - dest: /ostree/repo/config - owner: root - group: root - mode: 0644 - -- name: "WORK AROUND: Stat redhat repo file" - stat: path=/etc/yum.repos.d/redhat.repo - register: redhat_repo - -- name: "WORK AROUND: subscription manager failures" - file: - path: /etc/yum.repos.d/redhat.repo - state: touch - when: redhat_repo.stat.exists == False diff --git a/roles/atomic_base/tasks/system.yml b/roles/atomic_base/tasks/system.yml deleted file mode 100644 index e5cde427d..000000000 --- a/roles/atomic_base/tasks/system.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: Upload nofile limits.d file - copy: src=system/90-nofile.conf dest=/etc/security/limits.d/90-nofile.conf owner=root group=root mode=0644 diff --git a/roles/atomic_base/vars/main.yml b/roles/atomic_base/vars/main.yml deleted file mode 100644 index d4e61175c..000000000 --- a/roles/atomic_base/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -oo_timezone: US/Eastern diff --git a/roles/atomic_proxy/README.md b/roles/atomic_proxy/README.md deleted file mode 100644 index 348eaee1f..000000000 --- a/roles/atomic_proxy/README.md +++ /dev/null @@ -1,56 +0,0 @@ -Role Name -======== - -The purpose of this role is to do common configurations for all RHEL atomic hosts. - - -Requirements ------------- - -None - - -Role Variables --------------- - -None - - -Dependencies ------------- - -None - - -Example Playbook -------------------------- - -From a group playbook: - - hosts: servers - roles: - - ../../roles/atomic_proxy - - -License -------- - -Copyright 2012-2014 Red Hat, Inc., All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -Author Information ------------------- - -Thomas Wiest <twiest@redhat.com> diff --git a/roles/atomic_proxy/files/proxy_containers_deploy_descriptor.json b/roles/atomic_proxy/files/proxy_containers_deploy_descriptor.json deleted file mode 100644 index c15835d48..000000000 --- a/roles/atomic_proxy/files/proxy_containers_deploy_descriptor.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "Containers":[ - { - "Name":"proxy-puppet", - "Count":1, - "Image":"puppet:latest", - "PublicPorts":[ - ] - }, - { - "Name":"proxy", - "Count":1, - "Image":"proxy:latest", - "PublicPorts":[ - {"Internal":80,"External":80}, - {"Internal":443,"External":443}, - {"Internal":4999,"External":4999} - ] - }, - { - "Name":"proxy-monitoring", - "Count":1, - "Image":"monitoring:latest", - "PublicPorts":[ - ] - } - ], - "RandomizeIds": false -} diff --git a/roles/atomic_proxy/files/puppet/auth.conf b/roles/atomic_proxy/files/puppet/auth.conf deleted file mode 100644 index b31906bae..000000000 --- a/roles/atomic_proxy/files/puppet/auth.conf +++ /dev/null @@ -1,116 +0,0 @@ -# This is the default auth.conf file, which implements the default rules -# used by the puppet master. (That is, the rules below will still apply -# even if this file is deleted.) -# -# The ACLs are evaluated in top-down order. More specific stanzas should -# be towards the top of the file and more general ones at the bottom; -# otherwise, the general rules may "steal" requests that should be -# governed by the specific rules. -# -# See http://docs.puppetlabs.com/guides/rest_auth_conf.html for a more complete -# description of auth.conf's behavior. -# -# Supported syntax: -# Each stanza in auth.conf starts with a path to match, followed -# by optional modifiers, and finally, a series of allow or deny -# directives. -# -# Example Stanza -# --------------------------------- -# path /path/to/resource # simple prefix match -# # path ~ regex # alternately, regex match -# [environment envlist] -# [method methodlist] -# [auth[enthicated] {yes|no|on|off|any}] -# allow [host|backreference|*|regex] -# deny [host|backreference|*|regex] -# allow_ip [ip|cidr|ip_wildcard|*] -# deny_ip [ip|cidr|ip_wildcard|*] -# -# The path match can either be a simple prefix match or a regular -# expression. `path /file` would match both `/file_metadata` and -# `/file_content`. Regex matches allow the use of backreferences -# in the allow/deny directives. -# -# The regex syntax is the same as for Ruby regex, and captures backreferences -# for use in the `allow` and `deny` lines of that stanza -# -# Examples: -# -# path ~ ^/path/to/resource # Equivalent to `path /path/to/resource`. -# allow * # Allow all authenticated nodes (since auth -# # defaults to `yes`). -# -# path ~ ^/catalog/([^/]+)$ # Permit nodes to access their own catalog (by -# allow $1 # certname), but not any other node's catalog. -# -# path ~ ^/file_(metadata|content)/extra_files/ # Only allow certain nodes to -# auth yes # access the "extra_files" -# allow /^(.+)\.example\.com$/ # mount point; note this must -# allow_ip 192.168.100.0/24 # go ABOVE the "/file" rule, -# # since it is more specific. -# -# environment:: restrict an ACL to a comma-separated list of environments -# method:: restrict an ACL to a comma-separated list of HTTP methods -# auth:: restrict an ACL to an authenticated or unauthenticated request -# the default when unspecified is to restrict the ACL to authenticated requests -# (ie exactly as if auth yes was present). -# - -### Authenticated ACLs - these rules apply only when the client -### has a valid certificate and is thus authenticated - -# allow nodes to retrieve their own catalog -path ~ ^/catalog/([^/]+)$ -method find -allow $1 - -# allow nodes to retrieve their own node definition -path ~ ^/node/([^/]+)$ -method find -allow $1 - -# allow all nodes to access the certificates services -path /certificate_revocation_list/ca -method find -allow * - -# allow all nodes to store their own reports -path ~ ^/report/([^/]+)$ -method save -allow $1 - -# Allow all nodes to access all file services; this is necessary for -# pluginsync, file serving from modules, and file serving from custom -# mount points (see fileserver.conf). Note that the `/file` prefix matches -# requests to both the file_metadata and file_content paths. See "Examples" -# above if you need more granular access control for custom mount points. -path /file -allow * - -### Unauthenticated ACLs, for clients without valid certificates; authenticated -### clients can also access these paths, though they rarely need to. - -# allow access to the CA certificate; unauthenticated nodes need this -# in order to validate the puppet master's certificate -path /certificate/ca -auth any -method find -allow * - -# allow nodes to retrieve the certificate they requested earlier -path /certificate/ -auth any -method find -allow * - -# allow nodes to request a new certificate -path /certificate_request -auth any -method find, save -allow * - -# deny everything else; this ACL is not strictly necessary, but -# illustrates the default policy. -path / -auth any diff --git a/roles/atomic_proxy/files/setup-proxy-containers.sh b/roles/atomic_proxy/files/setup-proxy-containers.sh deleted file mode 100755 index d047c96c1..000000000 --- a/roles/atomic_proxy/files/setup-proxy-containers.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -function fail { - msg=$1 - echo - echo $msg - echo - exit 5 -} - - -NUM_DATA_CTR=$(docker ps -a | grep -c proxy-shared-data-1) -[ "$NUM_DATA_CTR" -ne 0 ] && fail "ERROR: proxy-shared-data-1 exists" - - -# pre-cache the container images -echo -timeout --signal TERM --kill-after 30 600 docker pull busybox:latest || fail "ERROR: docker pull of busybox failed" - -echo -# WORKAROUND: Setup the shared data container -/usr/bin/docker run --name "proxy-shared-data-1" \ - -v /shared/etc/haproxy \ - -v /shared/etc/httpd \ - -v /shared/etc/openshift \ - -v /shared/etc/pki \ - -v /shared/var/run/ctr-ipc \ - -v /shared/var/lib/haproxy \ - -v /shared/usr/local \ - "busybox:latest" true - -# WORKAROUND: These are because we're not using a pod yet -cp /usr/local/etc/ctr-proxy-1.service /usr/local/etc/ctr-proxy-puppet-1.service /usr/local/etc/ctr-proxy-monitoring-1.service /etc/systemd/system/ - -systemctl daemon-reload - -echo -echo -n "sleeping 10 seconds for systemd reload to take affect..." -sleep 10 -echo " Done." - -# Start the services -systemctl start ctr-proxy-puppet-1 ctr-proxy-1 ctr-proxy-monitoring-1 diff --git a/roles/atomic_proxy/handlers/main.yml b/roles/atomic_proxy/handlers/main.yml deleted file mode 100644 index 8eedec17a..000000000 --- a/roles/atomic_proxy/handlers/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: reload systemd - command: systemctl daemon-reload diff --git a/roles/atomic_proxy/meta/main.yml b/roles/atomic_proxy/meta/main.yml deleted file mode 100644 index a92d685b1..000000000 --- a/roles/atomic_proxy/meta/main.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -galaxy_info: - author: Thomas Wiest - description: Common base RHEL atomic configurations - company: Red Hat - # Some suggested licenses: - # - BSD (default) - # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: Apache - min_ansible_version: 1.2 - platforms: - - name: EL - versions: - - 7 -dependencies: - # This is the role's PRIVATE counterpart, which is used. - - ../../../../../atomic_private/ansible/roles/atomic_proxy diff --git a/roles/atomic_proxy/tasks/main.yml b/roles/atomic_proxy/tasks/main.yml deleted file mode 100644 index 073a1c61e..000000000 --- a/roles/atomic_proxy/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include: setup_puppet.yml -- include: setup_containers.yml diff --git a/roles/atomic_proxy/tasks/setup_containers.yml b/roles/atomic_proxy/tasks/setup_containers.yml deleted file mode 100644 index ee971623a..000000000 --- a/roles/atomic_proxy/tasks/setup_containers.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- name: "get output of: docker images" - command: docker images - changed_when: False # don't report as changed - register: docker_images - -- name: docker pull busybox ONLY if it's not present - command: "docker pull busybox:latest" - when: "not docker_images.stdout | search('busybox.*latest')" - -- name: docker pull containers ONLY if they're not present (needed otherwise systemd will timeout pulling the containers) - command: "docker pull docker-registry.ops.rhcloud.com/{{ item }}:{{ oo_env }}" - with_items: - - oso-v2-proxy - - oso-v2-puppet - - oso-v2-monitoring - when: "not docker_images.stdout | search('docker-registry.ops.rhcloud.com/{{ item }}.*{{ oo_env }}')" - -- name: "get output of: docker ps -a" - command: docker ps -a - changed_when: False # don't report as changed - register: docker_ps - -- name: run proxy-shared-data-1 - command: /usr/bin/docker run --name "proxy-shared-data-1" \ - -v /shared/etc/haproxy \ - -v /shared/etc/httpd \ - -v /shared/etc/openshift \ - -v /shared/etc/pki \ - -v /shared/var/run/ctr-ipc \ - -v /shared/var/lib/haproxy \ - -v /shared/usr/local \ - "busybox:latest" true - when: "not docker_ps.stdout | search('proxy-shared-data-1')" - -- name: Deploy systemd files for containers - template: - src: "systemd/{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - mode: 0640 - owner: root - group: root - with_items: - - ctr-proxy-1.service - - ctr-proxy-monitoring-1.service - - ctr-proxy-puppet-1.service - notify: reload systemd - -- name: start containers - service: - name: "{{ item }}" - state: started - enabled: yes - with_items: - - ctr-proxy-puppet-1 - - ctr-proxy-1 - - ctr-proxy-monitoring-1 diff --git a/roles/atomic_proxy/tasks/setup_puppet.yml b/roles/atomic_proxy/tasks/setup_puppet.yml deleted file mode 100644 index 7a599f06d..000000000 --- a/roles/atomic_proxy/tasks/setup_puppet.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: make puppet conf dir - file: - dest: "{{ oo_proxy_puppet_volume_dir }}/etc/puppet" - mode: 755 - owner: root - group: root - state: directory - -- name: upload puppet auth config - copy: - src: puppet/auth.conf - dest: "{{ oo_proxy_puppet_volume_dir }}/etc/puppet/auth.conf" - mode: 0644 - owner: root - group: root - -- name: upload puppet config - template: - src: puppet/puppet.conf.j2 - dest: "{{ oo_proxy_puppet_volume_dir }}/etc/puppet/puppet.conf" - mode: 0644 - owner: root - group: root diff --git a/roles/atomic_proxy/templates/puppet/puppet.conf.j2 b/roles/atomic_proxy/templates/puppet/puppet.conf.j2 deleted file mode 100644 index 9731ff168..000000000 --- a/roles/atomic_proxy/templates/puppet/puppet.conf.j2 +++ /dev/null @@ -1,40 +0,0 @@ -[main] - # we need to override the host name of the container - certname = ctr-proxy.{{ oo_env }}.rhcloud.com - - # The Puppet log directory. - # The default value is '$vardir/log'. - logdir = /var/log/puppet - - # Where Puppet PID files are kept. - # The default value is '$vardir/run'. - rundir = /var/run/puppet - - # Where SSL certificates are kept. - # The default value is '$confdir/ssl'. - ssldir = $vardir/ssl - manifest = $manifestdir/site.pp - manifestdir = /var/lib/puppet/environments/pub/$environment/manifests - environment = {{ oo_env_long }} - modulepath = /var/lib/puppet/environments/pub/$environment/modules:/var/lib/puppet/environments/pri/$environment/modules:/var/lib/puppet/environments/pri/production/modules:$confdir/modules:/usr/share/puppet/modules - -[agent] - # The file in which puppetd stores a list of the classes - # associated with the retrieved configuratiion. Can be loaded in - # the separate ``puppet`` executable using the ``--loadclasses`` - # option. - # The default value is '$confdir/classes.txt'. - classfile = $vardir/classes.txt - - # Where puppetd caches the local configuration. An - # extension indicating the cache format is added automatically. - # The default value is '$confdir/localconfig'. - localconfig = $vardir/localconfig - server = puppet.ops.rhcloud.com - environment = {{ oo_env_long }} - pluginsync = true - graph = true - configtimeout = 600 - report = true - runinterval = 3600 - splay = true diff --git a/roles/atomic_proxy/templates/sync/sync-proxy-configs.sh.j2 b/roles/atomic_proxy/templates/sync/sync-proxy-configs.sh.j2 deleted file mode 100755 index d9aa2d811..000000000 --- a/roles/atomic_proxy/templates/sync/sync-proxy-configs.sh.j2 +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -VOL_DIR=/var/lib/docker/volumes/proxy -SSH_CMD="ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null" - -mkdir -p ${VOL_DIR}/etc/haproxy/ -rsync -e "${SSH_CMD}" -va --progress root@proxy1.{{ oo_env }}.rhcloud.com:/etc/haproxy/ ${VOL_DIR}/etc/haproxy/ - -mkdir -p ${VOL_DIR}/etc/httpd/ -rsync -e "${SSH_CMD}" -va --progress root@proxy1.{{ oo_env }}.rhcloud.com:/etc/httpd/ ${VOL_DIR}/etc/httpd/ - -mkdir -p ${VOL_DIR}/etc/pki/tls/ -rsync -e "${SSH_CMD}" -va --progress root@proxy1.{{ oo_env }}.rhcloud.com:/etc/pki/tls/ ${VOL_DIR}/etc/pki/tls/ - -# We need to disable the haproxy chroot -sed -i -re 's/^(\s+)chroot/\1#chroot/' /var/lib/docker/volumes/proxy/etc/haproxy/haproxy.cfg diff --git a/roles/atomic_proxy/templates/systemd/ctr-proxy-1.service.j2 b/roles/atomic_proxy/templates/systemd/ctr-proxy-1.service.j2 deleted file mode 100644 index 988a9f544..000000000 --- a/roles/atomic_proxy/templates/systemd/ctr-proxy-1.service.j2 +++ /dev/null @@ -1,32 +0,0 @@ -[Unit] -Description=Container proxy-1 - - -[Service] -Type=simple -TimeoutStartSec=5m -Slice=container-small.slice - -ExecStartPre=-/usr/bin/docker rm "proxy-1" - -ExecStart=/usr/bin/docker run --rm --name "proxy-1" \ - --volumes-from proxy-shared-data-1 \ - -a stdout -a stderr -p 80:80 -p 443:443 -p 4999:4999 \ - "docker-registry.ops.rhcloud.com/oso-v2-proxy:{{ oo_env }}" - -ExecReload=-/usr/bin/docker stop "proxy-1" -ExecReload=-/usr/bin/docker rm "proxy-1" -ExecStop=-/usr/bin/docker stop "proxy-1" - -[Install] -WantedBy=container.target - -# Container information -X-ContainerId=proxy-1 -X-ContainerImage=docker-registry.ops.rhcloud.com/oso-v2-proxy:{{ oo_env }} -X-ContainerUserId= -X-ContainerRequestId=LwiWtYWaAvSavH6Ze53QJg -X-ContainerType=simple -X-PortMapping=80:80 -X-PortMapping=443:443 -X-PortMapping=4999:4999 diff --git a/roles/atomic_proxy/templates/systemd/ctr-proxy-monitoring-1.service.j2 b/roles/atomic_proxy/templates/systemd/ctr-proxy-monitoring-1.service.j2 deleted file mode 100644 index 975b0061b..000000000 --- a/roles/atomic_proxy/templates/systemd/ctr-proxy-monitoring-1.service.j2 +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -Description=Container proxy-monitoring-1 - - -[Service] -Type=simple -TimeoutStartSec=5m -Slice=container-small.slice - -ExecStartPre=-/usr/bin/docker rm "proxy-monitoring-1" - -ExecStart=/usr/bin/docker run --rm --name "proxy-monitoring-1" \ - --volumes-from proxy-shared-data-1 \ - -a stdout -a stderr \ - -e "OO_ENV={{ oo_env }}" \ - -e "OO_CTR_TYPE=proxy" \ - -e "OO_ZABBIX_HOSTGROUPS={{ oo_zabbix_hostgroups | join(',') }}" \ - -e "OO_ZABBIX_TEMPLATES=Template OpenShift Proxy Ctr" \ - "docker-registry.ops.rhcloud.com/oso-v2-monitoring:{{ oo_env }}" - -ExecReload=-/usr/bin/docker stop "proxy-monitoring-1" -ExecReload=-/usr/bin/docker rm "proxy-monitoring-1" -ExecStop=-/usr/bin/docker stop "proxy-monitoring-1" - -[Install] -WantedBy=container.target - -# Container information -X-ContainerId=proxy-monitoring-1 -X-ContainerImage=docker-registry.ops.rhcloud.com/oso-v2-monitoring:{{ oo_env }} -X-ContainerUserId= -X-ContainerRequestId=LwiWtYWaAvSavH6Ze53QJg -X-ContainerType=simple -X-PortMapping=80:80 -X-PortMapping=443:443 -X-PortMapping=4999:4999 diff --git a/roles/atomic_proxy/templates/systemd/ctr-proxy-puppet-1.service.j2 b/roles/atomic_proxy/templates/systemd/ctr-proxy-puppet-1.service.j2 deleted file mode 100644 index c3f28f471..000000000 --- a/roles/atomic_proxy/templates/systemd/ctr-proxy-puppet-1.service.j2 +++ /dev/null @@ -1,33 +0,0 @@ -[Unit] -Description=Container proxy-puppet-1 - - -[Service] -Type=simple -TimeoutStartSec=5m -Slice=container-small.slice - - -ExecStartPre=-/usr/bin/docker rm "proxy-puppet-1" - -ExecStart=/usr/bin/docker run --rm --name "proxy-puppet-1" \ - --volumes-from proxy-shared-data-1 \ - -v /var/lib/docker/volumes/proxy_puppet/var/lib/puppet/ssl:/var/lib/puppet/ssl \ - -v /var/lib/docker/volumes/proxy_puppet/etc/puppet:/etc/puppet \ - -a stdout -a stderr \ - "docker-registry.ops.rhcloud.com/oso-v2-puppet:{{ oo_env }}" - -# Set links (requires container have a name) -ExecReload=-/usr/bin/docker stop "proxy-puppet-1" -ExecReload=-/usr/bin/docker rm "proxy-puppet-1" -ExecStop=-/usr/bin/docker stop "proxy-puppet-1" - -[Install] -WantedBy=container.target - -# Container information -X-ContainerId=proxy-puppet-1 -X-ContainerImage=docker-registry.ops.rhcloud.com/oso-v2-puppet:{{ oo_env }} -X-ContainerUserId= -X-ContainerRequestId=Ky0lhw0onwoSDJR4GK6t3g -X-ContainerType=simple diff --git a/roles/atomic_proxy/vars/main.yml b/roles/atomic_proxy/vars/main.yml deleted file mode 100644 index 1f90492fd..000000000 --- a/roles/atomic_proxy/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -oo_proxy_puppet_volume_dir: /var/lib/docker/volumes/proxy_puppet diff --git a/roles/docker/files/enter-container.sh b/roles/docker/files/enter-container.sh deleted file mode 100755 index 7cf5b8d83..000000000 --- a/roles/docker/files/enter-container.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ] -then - echo - echo "Usage: $(basename $0) <container_name>" - echo - exit 1 -fi - -PID=$(docker inspect --format '{{.State.Pid}}' $1) - -nsenter --target $PID --mount --uts --ipc --net --pid diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml new file mode 100644 index 000000000..eca7419c1 --- /dev/null +++ b/roles/docker/handlers/main.yml @@ -0,0 +1,4 @@ +--- + +- name: restart docker + service: name=docker state=restarted diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index ca700db17..96949230d 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,15 +1,8 @@ --- # tasks file for docker - name: Install docker - yum: pkg=docker-io + yum: pkg=docker - name: enable and start the docker service service: name=docker enabled=yes state=started -- copy: src=enter-container.sh dest=/usr/local/bin/enter-container.sh mode=0755 - -# From the origin rpm there exists instructions on how to -# setup origin properly. The following steps come from there -- name: Change root to be in the Docker group - user: name=root groups=dockerroot append=yes - diff --git a/roles/docker_storage/README.md b/roles/docker_storage/README.md new file mode 100644 index 000000000..0d8f31afc --- /dev/null +++ b/roles/docker_storage/README.md @@ -0,0 +1,39 @@ +docker_storage +========= + +Configure docker_storage options +------------ + +None + +Role Variables +-------------- + +None + +Dependencies +------------ + +None + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role/docker_storage: + - key: df.fs + value: xfs + } + +License +------- + +ASL 2.0 + +Author Information +------------------ + +Openshift operations, Red Hat, Inc diff --git a/playbooks/aws/os2-atomic-proxy/vars.yml b/roles/docker_storage/defaults/main.yml index ed97d539c..ed97d539c 100644 --- a/playbooks/aws/os2-atomic-proxy/vars.yml +++ b/roles/docker_storage/defaults/main.yml diff --git a/roles/docker_storage/handlers/main.yml b/roles/docker_storage/handlers/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/roles/docker_storage/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/docker_storage/meta/main.yml b/roles/docker_storage/meta/main.yml new file mode 100644 index 000000000..a5d51cd3a --- /dev/null +++ b/roles/docker_storage/meta/main.yml @@ -0,0 +1,9 @@ +--- +galaxy_info: + author: Openshift + description: Setup docker_storage options + company: Red Hat, Inc + license: ASL 2.0 + min_ansible_version: 1.2 +dependencies: +- docker diff --git a/roles/docker_storage/tasks/main.yml b/roles/docker_storage/tasks/main.yml new file mode 100644 index 000000000..48a3fc208 --- /dev/null +++ b/roles/docker_storage/tasks/main.yml @@ -0,0 +1,37 @@ +--- +- lvg: + pvs: "{{ dst_device }}" + vg: "{{ dst_vg }}" + register: dst_lvg + +- lvol: + lv: data + vg: "{{ dst_vg }}" + size: 95%VG + register: dst_lvol_data + +- lvol: + lv: metadata + vg: "{{ dst_vg }}" + size: 5%VG + register: dst_lvol_metadata + + +- name: Update docker_storage options + lineinfile: + dest: /etc/sysconfig/docker-storage + backrefs: yes + regexp: "^(DOCKER_STORAGE_OPTIONS=)" + line: '\1 --storage-opt {{ dst_options | oo_combine_key_value("=") | join(" --storage-opt ") }}' + when: dst_options is defined and dst_options | length > 0 + register: dst_config + + +- name: Reload systemd units + command: systemctl daemon-reload + notify: + - restart docker + when: dst_config | changed or + dst_lvg | changed or + dst_lvol_data | changed or + dst_lvol_metadata | changed diff --git a/roles/docker_storage/vars/main.yml b/roles/docker_storage/vars/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/roles/docker_storage/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/roles/etcd/README.md b/roles/etcd/README.md deleted file mode 100644 index 225dd44b9..000000000 --- a/roles/etcd/README.md +++ /dev/null @@ -1,38 +0,0 @@ -Role Name -========= - -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. - -Role Variables --------------- - -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. - -Dependencies ------------- - -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } - -License -------- - -BSD - -Author Information ------------------- - -An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml deleted file mode 100644 index b897913f9..000000000 --- a/roles/etcd/handlers/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: restart etcd - service: name=etcd state=restarted diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml deleted file mode 100644 index c5c362c60..000000000 --- a/roles/etcd/meta/main.yml +++ /dev/null @@ -1,124 +0,0 @@ ---- -galaxy_info: - author: your name - description: - company: your company (optional) - # Some suggested licenses: - # - BSD (default) - # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: license (GPLv2, CC-BY, etc) - min_ansible_version: 1.2 - # - # Below are all platforms currently available. Just uncomment - # the ones that apply to your role. If you don't see your - # platform on this list, let us know and we'll get it added! - # - #platforms: - #- name: EL - # versions: - # - all - # - 5 - # - 6 - # - 7 - #- name: GenericUNIX - # versions: - # - all - # - any - #- name: Fedora - # versions: - # - all - # - 16 - # - 17 - # - 18 - # - 19 - # - 20 - #- name: opensuse - # versions: - # - all - # - 12.1 - # - 12.2 - # - 12.3 - # - 13.1 - # - 13.2 - #- name: Amazon - # versions: - # - all - # - 2013.03 - # - 2013.09 - #- name: GenericBSD - # versions: - # - all - # - any - #- name: FreeBSD - # versions: - # - all - # - 8.0 - # - 8.1 - # - 8.2 - # - 8.3 - # - 8.4 - # - 9.0 - # - 9.1 - # - 9.1 - # - 9.2 - #- name: Ubuntu - # versions: - # - all - # - lucid - # - maverick - # - natty - # - oneiric - # - precise - # - quantal - # - raring - # - saucy - # - trusty - #- name: SLES - # versions: - # - all - # - 10SP3 - # - 10SP4 - # - 11 - # - 11SP1 - # - 11SP2 - # - 11SP3 - #- name: GenericLinux - # versions: - # - all - # - any - #- name: Debian - # versions: - # - all - # - etch - # - lenny - # - squeeze - # - wheezy - # - # Below are all categories currently available. Just as with - # the platforms above, uncomment those that apply to your role. - # - #categories: - #- cloud - #- cloud:ec2 - #- cloud:gce - #- cloud:rax - #- clustering - #- database - #- database:nosql - #- database:sql - #- development - #- monitoring - #- networking - #- packaging - #- system - #- web -dependencies: [] - # List your role dependencies here, one per line. Only - # dependencies available via galaxy should be listed here. - # Be sure to remove the '[]' above if you add dependencies - # to this list. - diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml deleted file mode 100644 index 062d2e8a9..000000000 --- a/roles/etcd/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Install etcd - yum: pkg=etcd state=installed disable_gpg_check=yes - -- name: Install etcdctl - yum: pkg=etcdctl state=installed disable_gpg_check=yes - -- name: Write etcd global config file - template: src=etcd.conf.j2 dest=/etc/etcd/etcd.conf - notify: - - restart etcd - -- name: Open firewalld port for etcd - firewalld: port=4001/tcp permanent=false state=enabled - -- name: Save firewalld port for etcd - firewalld: port=4001/tcp permanent=true state=enabled - -- name: Enable etcd - service: name=etcd enabled=yes state=started diff --git a/roles/etcd/templates/etcd.conf.j2 b/roles/etcd/templates/etcd.conf.j2 deleted file mode 100644 index 1b43f6552..000000000 --- a/roles/etcd/templates/etcd.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -# This configuration file is written in [TOML](https://github.com/mojombo/toml) - -# addr = "127.0.0.1:4001" -# bind_addr = "127.0.0.1:4001" -# ca_file = "" -# cert_file = "" -# cors = [] -# cpu_profile_file = "" -# data_dir = "." -# discovery = "http://etcd.local:4001/v2/keys/_etcd/registry/examplecluster" -# http_read_timeout = 10 -# http_write_timeout = 10 -# key_file = "" -# peers = [] -# peers_file = "" -# max_cluster_size = 9 -# max_result_buffer = 1024 -# max_retry_attempts = 3 -# name = "default-name" -# snapshot = false -# verbose = false -# very_verbose = false - -# [peer] -# addr = "127.0.0.1:7001" -# bind_addr = "127.0.0.1:7001" -# ca_file = "" -# cert_file = "" -# key_file = "" - -# [cluster] -# active_size = 9 -# remove_delay = 1800.0 -# sync_interval = 5.0 diff --git a/roles/fluentd_master/tasks/main.yml b/roles/fluentd_master/tasks/main.yml new file mode 100644 index 000000000..28caaa5b8 --- /dev/null +++ b/roles/fluentd_master/tasks/main.yml @@ -0,0 +1,46 @@ +--- +# TODO: Update fluentd install and configuration when packaging is complete +- name: download and install td-agent + yum: + name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm' + state: present + +- name: Verify fluentd plugin installed + command: '/opt/td-agent/embedded/bin/gem query -i fluent-plugin-kubernetes' + register: _fluent_plugin_check + ignore_errors: yes + +- name: install Kubernetes fluentd plugin + command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes' + when: _fluent_plugin_check.rc == 1 + +- name: Creates directories + file: + path: "{{ item }}" + state: directory + group: 'td-agent' + owner: 'td-agent' + mode: 0755 + with_items: ['/etc/td-agent/config.d'] + +- name: Add include to td-agent configuration + lineinfile: + dest: '/etc/td-agent/td-agent.conf' + regexp: '^@include config.d' + line: '@include config.d/*.conf' + state: present + +- name: install Kubernetes fluentd configuration file + template: + src: kubernetes.conf.j2 + dest: /etc/td-agent/config.d/kubernetes.conf + group: 'td-agent' + owner: 'td-agent' + mode: 0444 + +- name: ensure td-agent is running + service: + name: 'td-agent' + state: started + enabled: yes + diff --git a/roles/fluentd_master/templates/kubernetes.conf.j2 b/roles/fluentd_master/templates/kubernetes.conf.j2 new file mode 100644 index 000000000..7b5c86062 --- /dev/null +++ b/roles/fluentd_master/templates/kubernetes.conf.j2 @@ -0,0 +1,9 @@ +<match kubernetes.**> + type file + path /var/log/td-agent/containers.log + time_slice_format %Y%m%d + time_slice_wait 10m + time_format %Y%m%dT%H%M%S%z + compress gzip + utc +</match> diff --git a/roles/fluentd_node/tasks/main.yml b/roles/fluentd_node/tasks/main.yml new file mode 100644 index 000000000..2526057cb --- /dev/null +++ b/roles/fluentd_node/tasks/main.yml @@ -0,0 +1,54 @@ +--- +# TODO: Update fluentd install and configuration when packaging is complete +- name: download and install td-agent + yum: + name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm' + state: present + +- name: Verify fluentd plugin installed + command: '/opt/td-agent/embedded/bin/gem query -i fluent-plugin-kubernetes' + register: _fluent_plugin_check + ignore_errors: yes + +- name: install Kubernetes fluentd plugin + command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes' + when: _fluent_plugin_check.rc == 1 + +- name: Override td-agent configuration file + template: + src: td-agent.j2 + dest: /etc/sysconfig/td-agent + group: 'td-agent' + owner: 'td-agent' + mode: 0444 + +- name: Creates directories + file: + path: "{{ item }}" + state: directory + group: 'td-agent' + owner: 'td-agent' + mode: 0755 + with_items: ['/etc/td-agent/config.d', '/var/log/td-agent/tmp'] + +- name: Add include to td-agent configuration + lineinfile: + dest: '/etc/td-agent/td-agent.conf' + regexp: '^@include config.d' + line: '@include config.d/*.conf' + state: present + +- name: install Kubernetes fluentd configuration file + template: + src: kubernetes.conf.j2 + dest: /etc/td-agent/config.d/kubernetes.conf + group: 'td-agent' + owner: 'td-agent' + mode: 0444 + +- name: ensure td-agent is running + service: + name: 'td-agent' + state: started + enabled: yes + diff --git a/roles/fluentd_node/templates/kubernetes.conf.j2 b/roles/fluentd_node/templates/kubernetes.conf.j2 new file mode 100644 index 000000000..5f1eecb20 --- /dev/null +++ b/roles/fluentd_node/templates/kubernetes.conf.j2 @@ -0,0 +1,53 @@ +<source> + type tail + path /var/lib/docker/containers/*/*-json.log + pos_file /var/log/td-agent/tmp/fluentd-docker.pos + time_format %Y-%m-%dT%H:%M:%S + tag docker.* + format json + read_from_head true +</source> + +<match docker.var.lib.docker.containers.*.*.log> + type kubernetes + container_id ${tag_parts[5]} + tag docker.${name} +</match> + +<match kubernetes> + type copy + + <store> + type forward + send_timeout 60s + recover_wait 10s + heartbeat_interval 1s + phi_threshold 16 + hard_timeout 60s + log_level trace + require_ack_response true + heartbeat_type tcp + + <server> + name {{groups['oo_first_master'][0]}} + host {{hostvars[groups['oo_first_master'][0]].openshift.common.hostname}} + port 24224 + weight 60 + </server> + + <secondary> + type file + path /var/log/td-agent/forward-failed + </secondary> + </store> + + <store> + type file + path /var/log/td-agent/containers.log + time_slice_format %Y%m%d + time_slice_wait 10m + time_format %Y%m%dT%H%M%S%z + compress gzip + utc + </store> +</match> diff --git a/roles/fluentd_node/templates/td-agent.j2 b/roles/fluentd_node/templates/td-agent.j2 new file mode 100644 index 000000000..7245e11ec --- /dev/null +++ b/roles/fluentd_node/templates/td-agent.j2 @@ -0,0 +1,2 @@ +DAEMON_ARGS= +TD_AGENT_ARGS="/usr/sbin/td-agent --log /var/log/td-agent/td-agent.log --use-v1-config" diff --git a/roles/kube_nfs_volumes/README.md b/roles/kube_nfs_volumes/README.md new file mode 100644 index 000000000..56c69c286 --- /dev/null +++ b/roles/kube_nfs_volumes/README.md @@ -0,0 +1,111 @@ +# kube_nfs_volumes + +This role is useful to export disks as set of Kubernetes persistent volumes. +It does so by partitioning the disks, creating ext4 filesystem on each +partition, mounting the partitions, exporting the mounts via NFS and adding +these NFS shares as NFS persistent volumes to existing Kubernetes installation. + +All partitions on given disks are used as the persistent volumes, including +already existing partitions! There should be no other data (such as operating +system) on the disks! + +## Requirements + +* Running Kubernetes with NFS persistent volume support (on a remote machine). + +* Works only on RHEL/Fedora-like distros. + +## Role Variables + +``` +# Options of NFS exports. +nfs_export_options: "*(rw,no_root_squash,insecure,no_subtree_check)" + +# Directory, where the created partitions should be mounted. They will be +# mounted as <mount_dir>/sda1 etc. +mount_dir: /exports + +# Comma-separated list of disks to partition. +# This role always assumes that all partitions on these disks are used as +# physical volumes. +disks: /dev/sdb,/dev/sdc + +# Whether to re-partition already partitioned disks. +# Even though the disks won't get repartitioned on 'false', all existing +# partitions on the disk are exported via NFS as physical volumes! +force: false + +# Specification of size of partitions to create. See library/partitionpool.py +# for details. +sizes: 100M + +# URL of Kubernetes API server, incl. port. +kubernetes_url: https://10.245.1.2:6443 + +# Token to use for authentication to the API server +kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX +``` + +## Dependencies + +None + +## Example Playbook + +With this playbook, `/dev/sdb` is partitioned into 100MiB partitions, all of +them are mounted into `/exports/sdb<N>` directory and all these directories +are exported via NFS and added as physical volumes to Kubernetes running at +`https://10.245.1.2:6443`. + + - hosts: servers + roles: + - role: kube_nfs_volumes + disks: "/dev/sdb" + sizes: 100M + kubernetes_url: https://10.245.1.2:6443 + kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX + +See library/partitionpool.py for details how `sizes` parameter can be used +to create partitions of various sizes. + +## Full example +Let's say there are two machines, 10.0.0.1 and 10.0.0.2, that we want to use as +NFS servers for our Kubernetes cluster, running Kubernetes public API at +https://10.245.1.2:6443. + +Both servers have three 1 TB disks, /dev/sda for the system and /dev/sdb and +/dev/sdc to be partitioned. We want to split the data disks into 5, 10 and +20 GiB partitions so that 10% of the total capacity is in 5 GiB partitions, 40% +in 10 GiB and 50% in 20 GiB partitions. + +That means, each data disk will have 20x 5 GiB, 40x 10 GiB and 25x 20 GiB +partitions. + +* Create an `inventory` file: + ``` + [nfsservers] + 10.0.0.1 + 10.0.0.2 + ``` + +* Create an ansible playbook, say `setupnfs.yaml`: + ``` + - hosts: nfsservers + sudo: yes + roles: + - role: kube_nfs_volumes + disks: "/dev/sdb,/dev/sdc" + sizes: 5G:10,10G:40,20G:50 + force: no + kubernetes_url: https://10.245.1.2:6443 + kubernetes_token: tJdce6Fn3cL1112YoIJ5m2exzAbzcPZX + ``` + +* Run the playbook: + ``` + ansible-playbook -i inventory setupnfs.yml + ``` + +## License + +Apache 2.0 diff --git a/roles/kube_nfs_volumes/defaults/main.yml b/roles/kube_nfs_volumes/defaults/main.yml new file mode 100644 index 000000000..e296492f9 --- /dev/null +++ b/roles/kube_nfs_volumes/defaults/main.yml @@ -0,0 +1,10 @@ +--- +# Options of NFS exports. +nfs_export_options: "*(rw,no_root_squash,insecure,no_subtree_check)" + +# Directory, where the created partitions should be mounted. They will be +# mounted as <mount_dir>/sda1 etc. +mount_dir: /exports + +# Force re-partitioning the disks +force: false diff --git a/roles/kube_nfs_volumes/handlers/main.yml b/roles/kube_nfs_volumes/handlers/main.yml new file mode 100644 index 000000000..52f3ceffe --- /dev/null +++ b/roles/kube_nfs_volumes/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart nfs + service: name=nfs-server state=restarted diff --git a/roles/kube_nfs_volumes/library/partitionpool.py b/roles/kube_nfs_volumes/library/partitionpool.py new file mode 100644 index 000000000..1ac8eed4d --- /dev/null +++ b/roles/kube_nfs_volumes/library/partitionpool.py @@ -0,0 +1,240 @@ +#!/usr/bin/python +""" +Ansible module for partitioning. +""" + +# There is no pyparted on our Jenkins worker +# pylint: disable=import-error +import parted + +DOCUMENTATION = """ +--- +module: partitionpool +short_description; Partition a disk into parititions. +description: + - Creates partitions on given disk based on partition sizes and their weights. + Unless 'force' option is set to True, it ignores already partitioned disks. + + When the disk is empty or 'force' is set to True, it always creates a new + GPT partition table on the disk. Then it creates number of partitions, based + on their weights. + + This module should be used when a system admin wants to split existing disk(s) + into pools of partitions of specific sizes. It is not intended as generic disk + partitioning module! + + Independent on 'force' parameter value and actual disk state, the task + always fills 'partition_pool' fact with all partitions on given disks, + together with their sizes (in bytes). E.g.: + partition_sizes = [ + { name: sda1, Size: 1048576000 }, + { name: sda2, Size: 1048576000 }, + { name: sdb1, Size: 1048576000 }, + ... + ] + +options: + disk: + description: + - Disk to partition. + size: + description: + - Sizes of partitions to create and their weights. Has form of: + <size1>[:<weigth1>][,<size2>[:<weight2>][,...]] + - Any <size> can end with 'm' or 'M' for megabyte, 'g/G' for gigabyte + and 't/T' for terabyte. Megabyte is used when no unit is specified. + - If <weight> is missing, 1.0 is used. + - From each specified partition <sizeX>, number of these partitions are + created so they occupy spaces represented by <weightX>, proportionally to + other weights. + + - Example 1: size=100G says, that the whole disk is split in number of 100 GiB + partitions. On 1 TiB disk, 10 partitions will be created. + + - Example 2: size=100G:1,10G:1 says that ratio of space occupied by 100 GiB + partitions and 10 GiB partitions is 1:1. Therefore, on 1 TiB disk, 500 GiB + will be split into five 100 GiB partition and 500 GiB will be split into fifty + 10GiB partitions. + - size=100G:1,10G:1 = 5x 100 GiB and 50x 10 GiB partitions (on 1 TiB disk). + + - Example 3: size=200G:1,100G:2 says that the ratio of space occupied by 200 GiB + partitions and 100GiB partition is 1:2. Therefore, on 1 TiB disk, 1/3 + (300 GiB) should be occupied by 200 GiB partitions. Only one fits there, + so only one is created (we always round nr. of partitions *down*). Teh rest + (800 GiB) is split into eight 100 GiB partitions, even though it's more + than 2/3 of total space - free space is always allocated as much as possible. + - size=200G:1,100G:2 = 1x 200 GiB and 8x 100 GiB partitions (on 1 TiB disk). + + - Example: size=200G:1,100G:1,50G:1 says that the ratio of space occupied by + 200 GiB, 100 GiB and 50 GiB partitions is 1:1:1. Therefore 1/3 of 1 TiB disk + is dedicated to 200 GiB partitions. Only one fits there and only one is + created. The rest (800 GiB) is distributed according to remaining weights: + 100 GiB vs 50 GiB is 1:1, we create four 100 GiB partitions (400 GiB in total) + and eight 50 GiB partitions (again, 400 GiB). + - size=200G:1,100G:1,50G:1 = 1x 200 GiB, 4x 100 GiB and 8x 50 GiB partitions + (on 1 TiB disk). + + force: + description: + - If True, it will always overwite partition table on the disk and create new one. + - If False (default), it won't change existing partition tables. + +""" + +# It's not class, it's more a simple struct with almost no functionality. +# pylint: disable=too-few-public-methods +class PartitionSpec(object): + """ Simple class to represent required partitions.""" + def __init__(self, size, weight): + """ Initialize the partition specifications.""" + # Size of the partitions + self.size = size + # Relative weight of this request + self.weight = weight + # Number of partitions to create, will be calculated later + self.count = -1 + + def set_count(self, count): + """ Set count of parititions of this specification. """ + self.count = count + +def assign_space(total_size, specs): + """ + Satisfy all the PartitionSpecs according to their weight. + In other words, calculate spec.count of all the specs. + """ + total_weight = 0.0 + for spec in specs: + total_weight += float(spec.weight) + + for spec in specs: + num_blocks = int((float(spec.weight) / total_weight) * (total_size / float(spec.size))) + spec.set_count(num_blocks) + total_size -= num_blocks * spec.size + total_weight -= spec.weight + +def partition(diskname, specs, force=False, check_mode=False): + """ + Create requested partitions. + Returns nr. of created partitions or 0 when the disk was already partitioned. + """ + count = 0 + + dev = parted.getDevice(diskname) + try: + disk = parted.newDisk(dev) + except parted.DiskException: + # unrecognizable format, treat as empty disk + disk = None + + if disk and len(disk.partitions) > 0 and not force: + print "skipping", diskname + return 0 + + # create new partition table, wiping all existing data + disk = parted.freshDisk(dev, 'gpt') + # calculate nr. of partitions of each size + assign_space(dev.getSize(), specs) + last_megabyte = 1 + for spec in specs: + for _ in range(spec.count): + # create the partition + start = parted.sizeToSectors(last_megabyte, "MiB", dev.sectorSize) + length = parted.sizeToSectors(spec.size, "MiB", dev.sectorSize) + geo = parted.Geometry(device=dev, start=start, length=length) + filesystem = parted.FileSystem(type='ext4', geometry=geo) + part = parted.Partition( + disk=disk, + type=parted.PARTITION_NORMAL, + fs=filesystem, + geometry=geo) + disk.addPartition(partition=part, constraint=dev.optimalAlignedConstraint) + last_megabyte += spec.size + count += 1 + try: + if not check_mode: + disk.commit() + except parted.IOException: + # partitions have been written, but we have been unable to inform the + # kernel of the change, probably because they are in use. + # Ignore it and hope for the best... + pass + return count + +def parse_spec(text): + """ Parse string with partition specification. """ + tokens = text.split(",") + specs = [] + for token in tokens: + if not ":" in token: + token += ":1" + + (sizespec, weight) = token.split(':') + weight = float(weight) # throws exception with reasonable error string + + units = {"m": 1, "g": 1 << 10, "t": 1 << 20, "p": 1 << 30} + unit = units.get(sizespec[-1].lower(), None) + if not unit: + # there is no unit specifier, it must be just the number + size = float(sizespec) + unit = 1 + else: + size = float(sizespec[:-1]) + spec = PartitionSpec(int(size * unit), weight) + specs.append(spec) + return specs + +def get_partitions(diskpath): + """ Return array of partition names for given disk """ + dev = parted.getDevice(diskpath) + disk = parted.newDisk(dev) + partitions = [] + for part in disk.partitions: + (_, _, pname) = part.path.rsplit("/") + partitions.append({"name": pname, "size": part.getLength() * dev.sectorSize}) + + return partitions + + +def main(): + """ Ansible module main method. """ + module = AnsibleModule( + argument_spec=dict( + disks=dict(required=True, type='str'), + force=dict(required=False, default="no", type='bool'), + sizes=dict(required=True, type='str') + ), + supports_check_mode=True, + ) + + disks = module.params['disks'] + force = module.params['force'] + if force is None: + force = False + sizes = module.params['sizes'] + + try: + specs = parse_spec(sizes) + except ValueError, ex: + err = "Error parsing sizes=" + sizes + ": " + str(ex) + module.fail_json(msg=err) + + partitions = [] + changed_count = 0 + for disk in disks.split(","): + try: + changed_count += partition(disk, specs, force, module.check_mode) + except Exception, ex: + err = "Error creating partitions on " + disk + ": " + str(ex) + raise + #module.fail_json(msg=err) + partitions += get_partitions(disk) + + module.exit_json(changed=(changed_count > 0), ansible_facts={"partition_pool": partitions}) + +# ignore pylint errors related to the module_utils import +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import +# import module snippets +from ansible.module_utils.basic import * +main() + diff --git a/roles/kube_nfs_volumes/meta/main.yml b/roles/kube_nfs_volumes/meta/main.yml new file mode 100644 index 000000000..eb71a7a1f --- /dev/null +++ b/roles/kube_nfs_volumes/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + author: Jan Safranek + description: Partition disks and use them as Kubernetes NFS physical volumes. + company: Red Hat, Inc. + license: license (Apache) + min_ansible_version: 1.4 + platforms: + - name: EL + versions: + - 7 + - name: Fedora + versions: + - all + categories: + - cloud diff --git a/roles/kube_nfs_volumes/tasks/main.yml b/roles/kube_nfs_volumes/tasks/main.yml new file mode 100644 index 000000000..23b228d32 --- /dev/null +++ b/roles/kube_nfs_volumes/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Install pyparted (RedHat/Fedora) + yum: name=pyparted,python-httplib2 state=installed + +- name: partition the drives + partitionpool: disks={{ disks }} force={{ force }} sizes={{ sizes }} + +- name: create filesystem + filesystem: fstype=ext4 dev=/dev/{{ item.name }} + with_items: partition_pool + +- name: mount + mount: name={{mount_dir}}/{{ item.name }} src=/dev/{{ item.name }} state=mounted fstype=ext4 passno=2 + with_items: partition_pool + +- include: nfs.yml + +- name: export physical volumes + uri: url={{ kubernetes_url }}/api/v1beta3/persistentvolumes + method=POST + body='{{ lookup("template", "../templates/nfs.json.j2") }}' + body_format=json + status_code=201 + HEADER_Authorization="Bearer {{ kubernetes_token }}" + with_items: partition_pool diff --git a/roles/kube_nfs_volumes/tasks/nfs.yml b/roles/kube_nfs_volumes/tasks/nfs.yml new file mode 100644 index 000000000..87cf5f9a4 --- /dev/null +++ b/roles/kube_nfs_volumes/tasks/nfs.yml @@ -0,0 +1,16 @@ +--- +- name: Install NFS server on Fedora/Red Hat + yum: name=nfs-utils state=installed + +- name: Start rpcbind on Fedora/Red Hat + service: name=rpcbind state=started enabled=yes + +- name: Start nfs on Fedora/Red Hat + service: name=nfs-server state=started enabled=yes + +- name: Export the directories + lineinfile: dest=/etc/exports + regexp="^{{ mount_dir }}/{{ item.name }} " + line="{{ mount_dir }}/{{ item.name }} {{nfs_export_options}}" + with_items: partition_pool + notify: restart nfs diff --git a/roles/kube_nfs_volumes/templates/nfs.json.j2 b/roles/kube_nfs_volumes/templates/nfs.json.j2 new file mode 100644 index 000000000..b42886ef1 --- /dev/null +++ b/roles/kube_nfs_volumes/templates/nfs.json.j2 @@ -0,0 +1,23 @@ +{ + "kind": "PersistentVolume", + "apiVersion": "v1beta3", + "metadata": { + "name": "pv-{{ inventory_hostname | regex_replace("\.", "-") }}-{{ item.name }}", + "labels": { + "type": "nfs" + } + }, + "spec": { + "capacity": { + "storage": "{{ item.size }}" + }, + "accessModes": [ + "ReadWriteOnce" + ], + "NFS": { + "Server": "{{ inventory_hostname }}", + "Path": "{{ mount_dir }}/{{ item.name }}", + "ReadOnly": false + } + } +} diff --git a/roles/openshift_ansible_inventory/tasks/main.yml b/roles/openshift_ansible_inventory/tasks/main.yml index dddfe24e3..5fe77e38b 100644 --- a/roles/openshift_ansible_inventory/tasks/main.yml +++ b/roles/openshift_ansible_inventory/tasks/main.yml @@ -24,22 +24,20 @@ owner: root group: libra_ops -- lineinfile: - dest: /etc/ansible/ansible.cfg - backrefs: yes - regexp: '^(hostfile|inventory)( *)=' - line: '\1\2= /etc/ansible/inventory' +# This cron uses the above location to call its job +- name: Cron to keep cache fresh + cron: + name: 'multi_ec2_inventory' + minute: '*/10' + job: '/usr/share/ansible/inventory/multi_ec2.py --refresh-cache &> /dev/null' + when: oo_cron_refresh_cache is defined and oo_cron_refresh_cache -- name: setting ec2.ini destination_format - lineinfile: - dest: /usr/share/ansible/inventory/aws/ec2.ini - regexp: '^destination_format *=' - line: "destination_format = {{ oo_ec2_destination_format }}" - when: oo_ec2_destination_format is defined - -- name: setting ec2.ini destination_format_tags - lineinfile: - dest: /usr/share/ansible/inventory/aws/ec2.ini - regexp: '^destination_format_tags *=' - line: "destination_format_tags = {{ oo_ec2_destination_format_tags }}" - when: oo_ec2_destination_format_tags is defined +- name: Set cache location + file: + state: directory + dest: "{{ oo_inventory_cache_location | dirname }}" + owner: root + group: libra_ops + recurse: yes + mode: '2750' + when: oo_inventory_cache_location is defined diff --git a/roles/openshift_ansible_inventory/templates/multi_ec2.yaml.j2 b/roles/openshift_ansible_inventory/templates/multi_ec2.yaml.j2 index 23dfe73b8..8228ab915 100644 --- a/roles/openshift_ansible_inventory/templates/multi_ec2.yaml.j2 +++ b/roles/openshift_ansible_inventory/templates/multi_ec2.yaml.j2 @@ -1,11 +1,26 @@ # multi ec2 inventory configs cache_max_age: {{ oo_inventory_cache_max_age }} +cache_location: {{ oo_inventory_cache_location | default('~/.ansible/tmp/multi_ec2_inventory.cache') }} accounts: {% for account in oo_inventory_accounts %} - name: {{ account.name }} provider: {{ account.provider }} + provider_config: +{% for section, items in account.provider_config.items() %} + {{ section }}: +{% for property, value in items.items() %} + {{ property }}: {{ value }} +{% endfor %} +{% endfor %} env_vars: AWS_ACCESS_KEY_ID: {{ account.env_vars.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: {{ account.env_vars.AWS_SECRET_ACCESS_KEY }} +{% if account.all_group is defined and account.hostvars is defined%} + all_group: {{ account.all_group }} + hostvars: +{% for property, value in account.hostvars.items() %} + {{ property }}: {{ value }} +{% endfor %} +{% endif %} {% endfor %} diff --git a/roles/openshift_common/tasks/main.yml b/roles/openshift_common/tasks/main.yml index c55677c3f..5bd8690a7 100644 --- a/roles/openshift_common/tasks/main.yml +++ b/roles/openshift_common/tasks/main.yml @@ -10,6 +10,7 @@ public_hostname: "{{ openshift_public_hostname | default(None) }}" public_ip: "{{ openshift_public_ip | default(None) }}" use_openshift_sdn: "{{ openshift_use_openshift_sdn | default(None) }}" + use_fluentd: "{{ openshift_use_fluentd | default(True) }}" deployment_type: "{{ openshift_deployment_type }}" - name: Set hostname hostname: name={{ openshift.common.hostname }} diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 1e0d5c605..9c2657ff2 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -1,6 +1,11 @@ #!/usr/bin/python # -*- coding: utf-8 -*- # vim: expandtab:tabstop=4:shiftwidth=4 +# disable pylint checks +# temporarily disabled until items can be addressed: +# fixme - until all TODO comments have been addressed +# pylint:disable=fixme +"""Ansible module for retrieving and setting openshift related facts""" DOCUMENTATION = ''' --- @@ -15,294 +20,645 @@ EXAMPLES = ''' import ConfigParser import copy -class OpenShiftFactsUnsupportedRoleError(Exception): - pass -class OpenShiftFactsFileWriteError(Exception): - pass +def hostname_valid(hostname): + """ Test if specified hostname should be considered valid -class OpenShiftFactsMetadataUnavailableError(Exception): - pass + Args: + hostname (str): hostname to test + Returns: + bool: True if valid, otherwise False + """ + if (not hostname or + hostname.startswith('localhost') or + hostname.endswith('localdomain') or + len(hostname.split('.')) < 2): + return False -class OpenShiftFacts(): - known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'dns'] + return True - def __init__(self, role, filename, local_facts): - self.changed = False - self.filename = filename - if role not in self.known_roles: - raise OpenShiftFactsUnsupportedRoleError("Role %s is not supported by this module" % role) - self.role = role - self.facts = self.generate_facts(local_facts) - def generate_facts(self, local_facts): - local_facts = self.init_local_facts(local_facts) - roles = local_facts.keys() +def choose_hostname(hostnames=None, fallback=''): + """ Choose a hostname from the provided hostnames - defaults = self.get_defaults(roles) - provider_facts = self.init_provider_facts() - facts = self.apply_provider_facts(defaults, provider_facts, roles) + Given a list of hostnames and a fallback value, choose a hostname to + use. This function will prefer fqdns if they exist (excluding any that + begin with localhost or end with localdomain) over ip addresses. - facts = self.merge_facts(facts, local_facts) - facts['current_config'] = self.current_config(facts) - self.set_url_facts_if_unset(facts) - return dict(openshift=facts) + Args: + hostnames (list): list of hostnames + fallback (str): default value to set if hostnames does not contain + a valid hostname + Returns: + str: chosen hostname + """ + hostname = fallback + if hostnames is None: + return hostname + + ip_regex = r'\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z' + ips = [i for i in hostnames + if (i is not None and isinstance(i, basestring) + and re.match(ip_regex, i))] + hosts = [i for i in hostnames + if i is not None and i != '' and i not in ips] + + for host_list in (hosts, ips): + for host in host_list: + if hostname_valid(host): + return host + + return hostname + + +def query_metadata(metadata_url, headers=None, expect_json=False): + """ Return metadata from the provided metadata_url + + Args: + metadata_url (str): metadata url + headers (dict): headers to set for metadata request + expect_json (bool): does the metadata_url return json + Returns: + dict or list: metadata request result + """ + result, info = fetch_url(module, metadata_url, headers=headers) + if info['status'] != 200: + raise OpenShiftFactsMetadataUnavailableError("Metadata unavailable") + if expect_json: + return module.from_json(result.read()) + else: + return [line.strip() for line in result.readlines()] + + +def walk_metadata(metadata_url, headers=None, expect_json=False): + """ Walk the metadata tree and return a dictionary of the entire tree + + Args: + metadata_url (str): metadata url + headers (dict): headers to set for metadata request + expect_json (bool): does the metadata_url return json + Returns: + dict: the result of walking the metadata tree + """ + metadata = dict() + + for line in query_metadata(metadata_url, headers, expect_json): + if line.endswith('/') and not line == 'public-keys/': + key = line[:-1] + metadata[key] = walk_metadata(metadata_url + line, + headers, expect_json) + else: + results = query_metadata(metadata_url + line, headers, + expect_json) + if len(results) == 1: + # disable pylint maybe-no-member because overloaded use of + # the module name causes pylint to not detect that results + # is an array or hash + # pylint: disable=maybe-no-member + metadata[line] = results.pop() + else: + metadata[line] = results + return metadata - def set_url_facts_if_unset(self, facts): - if 'master' in facts: - for (url_var, use_ssl, port, default) in [ - ('api_url', - facts['master']['api_use_ssl'], - facts['master']['api_port'], - facts['common']['hostname']), - ('public_api_url', - facts['master']['api_use_ssl'], - facts['master']['api_port'], - facts['common']['public_hostname']), - ('console_url', - facts['master']['console_use_ssl'], - facts['master']['console_port'], - facts['common']['hostname']), - ('public_console_url' 'console_use_ssl', - facts['master']['console_use_ssl'], - facts['master']['console_port'], - facts['common']['public_hostname'])]: - if url_var not in facts['master']: - scheme = 'https' if use_ssl else 'http' - netloc = default - if (scheme == 'https' and port != '443') or (scheme == 'http' and port != '80'): - netloc = "%s:%s" % (netloc, port) - facts['master'][url_var] = urlparse.urlunparse((scheme, netloc, '', '', '', '')) - - - # Query current OpenShift config and return a dictionary containing - # settings that may be valuable for determining actions that need to be - # taken in the playbooks/roles - def current_config(self, facts): - current_config=dict() - roles = [ role for role in facts if role not in ['common','provider'] ] - for role in roles: - if 'roles' in current_config: - current_config['roles'].append(role) +def get_provider_metadata(metadata_url, supports_recursive=False, + headers=None, expect_json=False): + """ Retrieve the provider metadata + + Args: + metadata_url (str): metadata url + supports_recursive (bool): does the provider metadata api support + recursion + headers (dict): headers to set for metadata request + expect_json (bool): does the metadata_url return json + Returns: + dict: the provider metadata + """ + try: + if supports_recursive: + metadata = query_metadata(metadata_url, headers, + expect_json) + else: + metadata = walk_metadata(metadata_url, headers, + expect_json) + except OpenShiftFactsMetadataUnavailableError: + metadata = None + return metadata + + +def normalize_gce_facts(metadata, facts): + """ Normalize gce facts + + Args: + metadata (dict): provider metadata + facts (dict): facts to update + Returns: + dict: the result of adding the normalized metadata to the provided + facts dict + """ + for interface in metadata['instance']['networkInterfaces']: + int_info = dict(ips=[interface['ip']], network_type='gce') + int_info['public_ips'] = [ac['externalIp'] for ac + in interface['accessConfigs']] + int_info['public_ips'].extend(interface['forwardedIps']) + _, _, network_id = interface['network'].rpartition('/') + int_info['network_id'] = network_id + facts['network']['interfaces'].append(int_info) + _, _, zone = metadata['instance']['zone'].rpartition('/') + facts['zone'] = zone + facts['external_id'] = metadata['instance']['id'] + + # Default to no sdn for GCE deployments + facts['use_openshift_sdn'] = False + + # GCE currently only supports a single interface + facts['network']['ip'] = facts['network']['interfaces'][0]['ips'][0] + pub_ip = facts['network']['interfaces'][0]['public_ips'][0] + facts['network']['public_ip'] = pub_ip + facts['network']['hostname'] = metadata['instance']['hostname'] + + # TODO: attempt to resolve public_hostname + facts['network']['public_hostname'] = facts['network']['public_ip'] + + return facts + + +def normalize_aws_facts(metadata, facts): + """ Normalize aws facts + + Args: + metadata (dict): provider metadata + facts (dict): facts to update + Returns: + dict: the result of adding the normalized metadata to the provided + facts dict + """ + for interface in sorted( + metadata['network']['interfaces']['macs'].values(), + key=lambda x: x['device-number'] + ): + int_info = dict() + var_map = {'ips': 'local-ipv4s', 'public_ips': 'public-ipv4s'} + for ips_var, int_var in var_map.iteritems(): + ips = interface.get(int_var) + if isinstance(ips, basestring): + int_info[ips_var] = [ips] else: - current_config['roles'] = [role] + int_info[ips_var] = ips + if 'vpc-id' in interface: + int_info['network_type'] = 'vpc' + else: + int_info['network_type'] = 'classic' + if int_info['network_type'] == 'vpc': + int_info['network_id'] = interface['subnet-id'] + else: + int_info['network_id'] = None + facts['network']['interfaces'].append(int_info) + facts['zone'] = metadata['placement']['availability-zone'] + facts['external_id'] = metadata['instance-id'] + + # TODO: actually attempt to determine default local and public ips + # by using the ansible default ip fact and the ipv4-associations + # from the ec2 metadata + facts['network']['ip'] = metadata.get('local-ipv4') + facts['network']['public_ip'] = metadata.get('public-ipv4') + + # TODO: verify that local hostname makes sense and is resolvable + facts['network']['hostname'] = metadata.get('local-hostname') + + # TODO: verify that public hostname makes sense and is resolvable + facts['network']['public_hostname'] = metadata.get('public-hostname') + + return facts + + +def normalize_openstack_facts(metadata, facts): + """ Normalize openstack facts + + Args: + metadata (dict): provider metadata + facts (dict): facts to update + Returns: + dict: the result of adding the normalized metadata to the provided + facts dict + """ + # openstack ec2 compat api does not support network interfaces and + # the version tested on did not include the info in the openstack + # metadata api, should be updated if neutron exposes this. + + facts['zone'] = metadata['availability_zone'] + facts['external_id'] = metadata['uuid'] + facts['network']['ip'] = metadata['ec2_compat']['local-ipv4'] + facts['network']['public_ip'] = metadata['ec2_compat']['public-ipv4'] + + # TODO: verify local hostname makes sense and is resolvable + facts['network']['hostname'] = metadata['hostname'] + + # TODO: verify that public hostname makes sense and is resolvable + pub_h = metadata['ec2_compat']['public-hostname'] + facts['network']['public_hostname'] = pub_h + + return facts + + +def normalize_provider_facts(provider, metadata): + """ Normalize provider facts + + Args: + provider (str): host provider + metadata (dict): provider metadata + Returns: + dict: the normalized provider facts + """ + if provider is None or metadata is None: + return {} + + # TODO: test for ipv6_enabled where possible (gce, aws do not support) + # and configure ipv6 facts if available + + # TODO: add support for setting user_data if available + + facts = dict(name=provider, metadata=metadata, + network=dict(interfaces=[], ipv6_enabled=False)) + if provider == 'gce': + facts = normalize_gce_facts(metadata, facts) + elif provider == 'ec2': + facts = normalize_aws_facts(metadata, facts) + elif provider == 'openstack': + facts = normalize_openstack_facts(metadata, facts) + return facts + + +def set_url_facts_if_unset(facts): + """ Set url facts if not already present in facts dict + + Args: + facts (dict): existing facts + Returns: + dict: the facts dict updated with the generated url facts if they + were not already present + """ + if 'master' in facts: + for (url_var, use_ssl, port, default) in [ + ('api_url', + facts['master']['api_use_ssl'], + facts['master']['api_port'], + facts['common']['hostname']), + ('public_api_url', + facts['master']['api_use_ssl'], + facts['master']['api_port'], + facts['common']['public_hostname']), + ('console_url', + facts['master']['console_use_ssl'], + facts['master']['console_port'], + facts['common']['hostname']), + ('public_console_url' 'console_use_ssl', + facts['master']['console_use_ssl'], + facts['master']['console_port'], + facts['common']['public_hostname'])]: + if url_var not in facts['master']: + scheme = 'https' if use_ssl else 'http' + netloc = default + if ((scheme == 'https' and port != '443') + or (scheme == 'http' and port != '80')): + netloc = "%s:%s" % (netloc, port) + facts['master'][url_var] = urlparse.urlunparse( + (scheme, netloc, '', '', '', '') + ) + return facts + + +def get_current_config(facts): + """ Get current openshift config + + Args: + facts (dict): existing facts + Returns: + dict: the facts dict updated with the current openshift config + """ + current_config = dict() + roles = [role for role in facts if role not in ['common', 'provider']] + for role in roles: + if 'roles' in current_config: + current_config['roles'].append(role) + else: + current_config['roles'] = [role] - # TODO: parse the /etc/sysconfig/openshift-{master,node} config to - # determine the location of files. + # TODO: parse the /etc/sysconfig/openshift-{master,node} config to + # determine the location of files. - # Query kubeconfig settings - kubeconfig_dir = '/var/lib/openshift/openshift.local.certificates' - if role == 'node': - kubeconfig_dir = os.path.join(kubeconfig_dir, "node-%s" % facts['common']['hostname']) + # Query kubeconfig settings + kubeconfig_dir = '/var/lib/openshift/openshift.local.certificates' + if role == 'node': + kubeconfig_dir = os.path.join( + kubeconfig_dir, "node-%s" % facts['common']['hostname'] + ) - kubeconfig_path = os.path.join(kubeconfig_dir, '.kubeconfig') - if os.path.isfile('/usr/bin/openshift') and os.path.isfile(kubeconfig_path): + kubeconfig_path = os.path.join(kubeconfig_dir, '.kubeconfig') + if (os.path.isfile('/usr/bin/openshift') + and os.path.isfile(kubeconfig_path)): + try: + _, output, _ = module.run_command( + ["/usr/bin/openshift", "ex", "config", "view", "-o", + "json", "--kubeconfig=%s" % kubeconfig_path], + check_rc=False + ) + config = json.loads(output) + + cad = 'certificate-authority-data' + try: + for cluster in config['clusters']: + config['clusters'][cluster][cad] = 'masked' + except KeyError: + pass try: - _, output, error = module.run_command(["/usr/bin/openshift", "ex", - "config", "view", "-o", - "json", - "--kubeconfig=%s" % kubeconfig_path], - check_rc=False) - config = json.loads(output) - - try: - for cluster in config['clusters']: - config['clusters'][cluster]['certificate-authority-data'] = 'masked' - except KeyError: - pass - try: - for user in config['users']: - config['users'][user]['client-certificate-data'] = 'masked' - config['users'][user]['client-key-data'] = 'masked' - except KeyError: - pass - - current_config['kubeconfig'] = config - except Exception: + for user in config['users']: + config['users'][user][cad] = 'masked' + config['users'][user]['client-key-data'] = 'masked' + except KeyError: pass - return current_config + current_config['kubeconfig'] = config + # override pylint broad-except warning, since we do not want + # to bubble up any exceptions if openshift ex config view + # fails + # pylint: disable=broad-except + except Exception: + pass - def apply_provider_facts(self, facts, provider_facts, roles): - if not provider_facts: - return facts + return current_config - use_openshift_sdn = provider_facts.get('use_openshift_sdn') - if isinstance(use_openshift_sdn, bool): - facts['common']['use_openshift_sdn'] = use_openshift_sdn - common_vars = [('hostname', 'ip'), ('public_hostname', 'public_ip')] - for h_var, ip_var in common_vars: - ip_value = provider_facts['network'].get(ip_var) - if ip_value: - facts['common'][ip_var] = ip_value +def apply_provider_facts(facts, provider_facts, roles): + """ Apply provider facts to supplied facts dict - facts['common'][h_var] = self.choose_hostname([provider_facts['network'].get(h_var)], facts['common'][ip_var]) + Args: + facts (dict): facts dict to update + provider_facts (dict): provider facts to apply + roles: host roles + Returns: + dict: the merged facts + """ + if not provider_facts: + return facts - if 'node' in roles: - ext_id = provider_facts.get('external_id') - if ext_id: - facts['node']['external_id'] = ext_id + use_openshift_sdn = provider_facts.get('use_openshift_sdn') + if isinstance(use_openshift_sdn, bool): + facts['common']['use_openshift_sdn'] = use_openshift_sdn - facts['provider'] = provider_facts - return facts + common_vars = [('hostname', 'ip'), ('public_hostname', 'public_ip')] + for h_var, ip_var in common_vars: + ip_value = provider_facts['network'].get(ip_var) + if ip_value: + facts['common'][ip_var] = ip_value - def hostname_valid(self, hostname): - if (not hostname or - hostname.startswith('localhost') or - hostname.endswith('localdomain') or - len(hostname.split('.')) < 2): - return False + facts['common'][h_var] = choose_hostname( + [provider_facts['network'].get(h_var)], + facts['common'][ip_var] + ) - return True + if 'node' in roles: + ext_id = provider_facts.get('external_id') + if ext_id: + facts['node']['external_id'] = ext_id + + facts['provider'] = provider_facts + return facts + + +def merge_facts(orig, new): + """ Recursively merge facts dicts + + Args: + orig (dict): existing facts + new (dict): facts to update + Returns: + dict: the merged facts + """ + facts = dict() + for key, value in orig.iteritems(): + if key in new: + if isinstance(value, dict): + facts[key] = merge_facts(value, new[key]) + else: + facts[key] = copy.copy(new[key]) + else: + facts[key] = copy.deepcopy(value) + new_keys = set(new.keys()) - set(orig.keys()) + for key in new_keys: + facts[key] = copy.deepcopy(new[key]) + return facts + + +def save_local_facts(filename, facts): + """ Save local facts + + Args: + filename (str): local facts file + facts (dict): facts to set + """ + try: + fact_dir = os.path.dirname(filename) + if not os.path.exists(fact_dir): + os.makedirs(fact_dir) + with open(filename, 'w') as fact_file: + fact_file.write(module.jsonify(facts)) + except (IOError, OSError) as ex: + raise OpenShiftFactsFileWriteError( + "Could not create fact file: %s, error: %s" % (filename, ex) + ) - def choose_hostname(self, hostnames=[], fallback=''): - hostname = fallback - ips = [ i for i in hostnames if i is not None and re.match(r'\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z', i) ] - hosts = [ i for i in hostnames if i is not None and i not in set(ips) ] +def get_local_facts_from_file(filename): + """ Retrieve local facts from fact file + + Args: + filename (str): local facts file + Returns: + dict: the retrieved facts + """ + local_facts = dict() + try: + # Handle conversion of INI style facts file to json style + ini_facts = ConfigParser.SafeConfigParser() + ini_facts.read(filename) + for section in ini_facts.sections(): + local_facts[section] = dict() + for key, value in ini_facts.items(section): + local_facts[section][key] = value + + except (ConfigParser.MissingSectionHeaderError, + ConfigParser.ParsingError): + try: + with open(filename, 'r') as facts_file: + local_facts = json.load(facts_file) + except (ValueError, IOError): + pass - for host_list in (hosts, ips): - for h in host_list: - if self.hostname_valid(h): - return h + return local_facts - return hostname + +class OpenShiftFactsUnsupportedRoleError(Exception): + """OpenShift Facts Unsupported Role Error""" + pass + + +class OpenShiftFactsFileWriteError(Exception): + """OpenShift Facts File Write Error""" + pass + + +class OpenShiftFactsMetadataUnavailableError(Exception): + """OpenShift Facts Metadata Unavailable Error""" + pass + + +class OpenShiftFacts(object): + """ OpenShift Facts + + Attributes: + facts (dict): OpenShift facts for the host + + Args: + role (str): role for setting local facts + filename (str): local facts file to use + local_facts (dict): local facts to set + + Raises: + OpenShiftFactsUnsupportedRoleError: + """ + known_roles = ['common', 'master', 'node', 'master_sdn', 'node_sdn', 'dns'] + + def __init__(self, role, filename, local_facts): + self.changed = False + self.filename = filename + if role not in self.known_roles: + raise OpenShiftFactsUnsupportedRoleError( + "Role %s is not supported by this module" % role + ) + self.role = role + self.system_facts = ansible_facts(module) + self.facts = self.generate_facts(local_facts) + + def generate_facts(self, local_facts): + """ Generate facts + + Args: + local_facts (dict): local_facts for overriding generated + defaults + + Returns: + dict: The generated facts + """ + local_facts = self.init_local_facts(local_facts) + roles = local_facts.keys() + + defaults = self.get_defaults(roles) + provider_facts = self.init_provider_facts() + facts = apply_provider_facts(defaults, provider_facts, roles) + facts = merge_facts(facts, local_facts) + facts['current_config'] = get_current_config(facts) + facts = set_url_facts_if_unset(facts) + return dict(openshift=facts) def get_defaults(self, roles): - ansible_facts = self.get_ansible_facts() + """ Get default fact values + Args: + roles (list): list of roles for this host + + Returns: + dict: The generated default facts + """ defaults = dict() common = dict(use_openshift_sdn=True) - ip = ansible_facts['default_ipv4']['address'] - common['ip'] = ip - common['public_ip'] = ip + ip_addr = self.system_facts['default_ipv4']['address'] + common['ip'] = ip_addr + common['public_ip'] = ip_addr - rc, output, error = module.run_command(['hostname', '-f']) - hostname_f = output.strip() if rc == 0 else '' - hostname_values = [hostname_f, ansible_facts['nodename'], ansible_facts['fqdn']] - hostname = self.choose_hostname(hostname_values) + exit_code, output, _ = module.run_command(['hostname', '-f']) + hostname_f = output.strip() if exit_code == 0 else '' + hostname_values = [hostname_f, self.system_facts['nodename'], + self.system_facts['fqdn']] + hostname = choose_hostname(hostname_values) common['hostname'] = hostname common['public_hostname'] = hostname defaults['common'] = common if 'master' in roles: - # TODO: provide for a better way to override just the port, or just - # the urls, instead of forcing both, also to override the hostname - # without having to re-generate these urls later master = dict(api_use_ssl=True, api_port='8443', - console_use_ssl=True, console_path='/console', - console_port='8443', etcd_use_ssl=False, - etcd_port='4001', portal_net='172.30.17.0/24') + console_use_ssl=True, console_path='/console', + console_port='8443', etcd_use_ssl=False, + etcd_port='4001', portal_net='172.30.17.0/24') defaults['master'] = master if 'node' in roles: node = dict(external_id=common['hostname'], pod_cidr='', labels={}, annotations={}) - node['resources_cpu'] = ansible_facts['processor_cores'] - node['resources_memory'] = int(int(ansible_facts['memtotal_mb']) * 1024 * 1024 * 0.75) + node['resources_cpu'] = self.system_facts['processor_cores'] + node['resources_memory'] = int( + int(self.system_facts['memtotal_mb']) * 1024 * 1024 * 0.75 + ) defaults['node'] = node return defaults - def merge_facts(self, orig, new): - facts = dict() - for key, value in orig.iteritems(): - if key in new: - if isinstance(value, dict): - facts[key] = self.merge_facts(value, new[key]) - else: - facts[key] = copy.copy(new[key]) - else: - facts[key] = copy.deepcopy(value) - new_keys = set(new.keys()) - set(orig.keys()) - for key in new_keys: - facts[key] = copy.deepcopy(new[key]) - return facts - - def query_metadata(self, metadata_url, headers=None, expect_json=False): - r, info = fetch_url(module, metadata_url, headers=headers) - if info['status'] != 200: - raise OpenShiftFactsMetadataUnavailableError("Metadata unavailable") - if expect_json: - return module.from_json(r.read()) - else: - return [line.strip() for line in r.readlines()] - - def walk_metadata(self, metadata_url, headers=None, expect_json=False): - metadata = dict() - - for line in self.query_metadata(metadata_url, headers, expect_json): - if line.endswith('/') and not line == 'public-keys/': - key = line[:-1] - metadata[key]=self.walk_metadata(metadata_url + line, headers, - expect_json) - else: - results = self.query_metadata(metadata_url + line, headers, - expect_json) - if len(results) == 1: - metadata[line] = results.pop() - else: - metadata[line] = results - return metadata - - def get_provider_metadata(self, metadata_url, supports_recursive=False, - headers=None, expect_json=False): - try: - if supports_recursive: - metadata = self.query_metadata(metadata_url, headers, expect_json) - else: - metadata = self.walk_metadata(metadata_url, headers, expect_json) - except OpenShiftFactsMetadataUnavailableError as e: - metadata = None - return metadata - - def get_ansible_facts(self): - if not hasattr(self, 'ansible_facts'): - self.ansible_facts = ansible_facts(module) - return self.ansible_facts - def guess_host_provider(self): + """ Guess the host provider + + Returns: + dict: The generated default facts for the detected provider + """ # TODO: cloud provider facts should probably be submitted upstream - ansible_facts = self.get_ansible_facts() - product_name = ansible_facts['product_name'] - product_version = ansible_facts['product_version'] - virt_type = ansible_facts['virtualization_type'] - virt_role = ansible_facts['virtualization_role'] + product_name = self.system_facts['product_name'] + product_version = self.system_facts['product_version'] + virt_type = self.system_facts['virtualization_type'] + virt_role = self.system_facts['virtualization_role'] provider = None metadata = None # TODO: this is not exposed through module_utils/facts.py in ansible, # need to create PR for ansible to expose it - bios_vendor = get_file_content('/sys/devices/virtual/dmi/id/bios_vendor') + bios_vendor = get_file_content( + '/sys/devices/virtual/dmi/id/bios_vendor' + ) if bios_vendor == 'Google': provider = 'gce' - metadata_url = 'http://metadata.google.internal/computeMetadata/v1/?recursive=true' + metadata_url = ('http://metadata.google.internal/' + 'computeMetadata/v1/?recursive=true') headers = {'Metadata-Flavor': 'Google'} - metadata = self.get_provider_metadata(metadata_url, True, headers, - True) + metadata = get_provider_metadata(metadata_url, True, headers, + True) # Filter sshKeys and serviceAccounts from gce metadata if metadata: metadata['project']['attributes'].pop('sshKeys', None) metadata['instance'].pop('serviceAccounts', None) - elif virt_type == 'xen' and virt_role == 'guest' and re.match(r'.*\.amazon$', product_version): + elif (virt_type == 'xen' and virt_role == 'guest' + and re.match(r'.*\.amazon$', product_version)): provider = 'ec2' metadata_url = 'http://169.254.169.254/latest/meta-data/' - metadata = self.get_provider_metadata(metadata_url) + metadata = get_provider_metadata(metadata_url) elif re.search(r'OpenStack', product_name): provider = 'openstack' - metadata_url = 'http://169.254.169.254/openstack/latest/meta_data.json' - metadata = self.get_provider_metadata(metadata_url, True, None, True) + metadata_url = ('http://169.254.169.254/openstack/latest/' + 'meta_data.json') + metadata = get_provider_metadata(metadata_url, True, None, + True) if metadata: ec2_compat_url = 'http://169.254.169.254/latest/meta-data/' - metadata['ec2_compat'] = self.get_provider_metadata(ec2_compat_url) - + metadata['ec2_compat'] = get_provider_metadata( + ec2_compat_url + ) + + # disable pylint maybe-no-member because overloaded use of + # the module name causes pylint to not detect that results + # is an array or hash + # pylint: disable=maybe-no-member # Filter public_keys and random_seed from openstack metadata metadata.pop('public_keys', None) metadata.pop('random_seed', None) @@ -312,146 +668,74 @@ class OpenShiftFacts(): return dict(name=provider, metadata=metadata) - def normalize_provider_facts(self, provider, metadata): - if provider is None or metadata is None: - return {} - - # TODO: test for ipv6_enabled where possible (gce, aws do not support) - # and configure ipv6 facts if available - - # TODO: add support for setting user_data if available - - facts = dict(name=provider, metadata=metadata) - network = dict(interfaces=[], ipv6_enabled=False) - if provider == 'gce': - for interface in metadata['instance']['networkInterfaces']: - int_info = dict(ips=[interface['ip']], network_type=provider) - int_info['public_ips'] = [ ac['externalIp'] for ac in interface['accessConfigs'] ] - int_info['public_ips'].extend(interface['forwardedIps']) - _, _, network_id = interface['network'].rpartition('/') - int_info['network_id'] = network_id - network['interfaces'].append(int_info) - _, _, zone = metadata['instance']['zone'].rpartition('/') - facts['zone'] = zone - facts['external_id'] = metadata['instance']['id'] - - # Default to no sdn for GCE deployments - facts['use_openshift_sdn'] = False - - # GCE currently only supports a single interface - network['ip'] = network['interfaces'][0]['ips'][0] - network['public_ip'] = network['interfaces'][0]['public_ips'][0] - network['hostname'] = metadata['instance']['hostname'] - - # TODO: attempt to resolve public_hostname - network['public_hostname'] = network['public_ip'] - elif provider == 'ec2': - for interface in sorted(metadata['network']['interfaces']['macs'].values(), - key=lambda x: x['device-number']): - int_info = dict() - var_map = {'ips': 'local-ipv4s', 'public_ips': 'public-ipv4s'} - for ips_var, int_var in var_map.iteritems(): - ips = interface[int_var] - int_info[ips_var] = [ips] if isinstance(ips, basestring) else ips - int_info['network_type'] = 'vpc' if 'vpc-id' in interface else 'classic' - int_info['network_id'] = interface['subnet-id'] if int_info['network_type'] == 'vpc' else None - network['interfaces'].append(int_info) - facts['zone'] = metadata['placement']['availability-zone'] - facts['external_id'] = metadata['instance-id'] - - # TODO: actually attempt to determine default local and public ips - # by using the ansible default ip fact and the ipv4-associations - # form the ec2 metadata - network['ip'] = metadata['local-ipv4'] - network['public_ip'] = metadata['public-ipv4'] - - # TODO: verify that local hostname makes sense and is resolvable - network['hostname'] = metadata['local-hostname'] - - # TODO: verify that public hostname makes sense and is resolvable - network['public_hostname'] = metadata['public-hostname'] - elif provider == 'openstack': - # openstack ec2 compat api does not support network interfaces and - # the version tested on did not include the info in the openstack - # metadata api, should be updated if neutron exposes this. - - facts['zone'] = metadata['availability_zone'] - facts['external_id'] = metadata['uuid'] - network['ip'] = metadata['ec2_compat']['local-ipv4'] - network['public_ip'] = metadata['ec2_compat']['public-ipv4'] - - # TODO: verify local hostname makes sense and is resolvable - network['hostname'] = metadata['hostname'] - - # TODO: verify that public hostname makes sense and is resolvable - network['public_hostname'] = metadata['ec2_compat']['public-hostname'] - - facts['network'] = network - return facts - def init_provider_facts(self): + """ Initialize the provider facts + + Returns: + dict: The normalized provider facts + """ provider_info = self.guess_host_provider() - provider_facts = self.normalize_provider_facts( - provider_info.get('name'), - provider_info.get('metadata') + provider_facts = normalize_provider_facts( + provider_info.get('name'), + provider_info.get('metadata') ) return provider_facts - def get_facts(self): - # TODO: transform facts into cleaner format (openshift_<blah> instead - # of openshift.<blah> - return self.facts - - def init_local_facts(self, facts={}): - changed = False + def init_local_facts(self, facts=None): + """ Initialize the provider facts - local_facts = ConfigParser.SafeConfigParser() - local_facts.read(self.filename) + Args: + facts (dict): local facts to set - section = self.role - if not local_facts.has_section(section): - local_facts.add_section(section) + Returns: + dict: The result of merging the provided facts with existing + local facts + """ + changed = False + facts_to_set = {self.role: dict()} + if facts is not None: + facts_to_set[self.role] = facts + + local_facts = get_local_facts_from_file(self.filename) + + for arg in ['labels', 'annotations']: + if arg in facts_to_set and isinstance(facts_to_set[arg], + basestring): + facts_to_set[arg] = module.from_json(facts_to_set[arg]) + + new_local_facts = merge_facts(local_facts, facts_to_set) + for facts in new_local_facts.values(): + keys_to_delete = [] + for fact, value in facts.iteritems(): + if value == "" or value is None: + keys_to_delete.append(fact) + for key in keys_to_delete: + del facts[key] + + if new_local_facts != local_facts: changed = True - for key, value in facts.iteritems(): - if isinstance(value, bool): - value = str(value) - if not value: - continue - if not local_facts.has_option(section, key) or local_facts.get(section, key) != value: - local_facts.set(section, key, value) - changed = True + if not module.check_mode: + save_local_facts(self.filename, new_local_facts) - if changed and not module.check_mode: - try: - fact_dir = os.path.dirname(self.filename) - if not os.path.exists(fact_dir): - os.makedirs(fact_dir) - with open(self.filename, 'w') as fact_file: - local_facts.write(fact_file) - except (IOError, OSError) as e: - raise OpenShiftFactsFileWriteError("Could not create fact file: %s, error: %s" % (self.filename, e)) self.changed = changed - - role_facts = dict() - for section in local_facts.sections(): - role_facts[section] = dict() - for opt, val in local_facts.items(section): - role_facts[section][opt] = val - return role_facts + return new_local_facts def main(): + """ main """ + # disabling pylint errors for global-variable-undefined and invalid-name + # for 'global module' usage, since it is required to use ansible_facts + # pylint: disable=global-variable-undefined, invalid-name global module module = AnsibleModule( - argument_spec = dict( - role=dict(default='common', - choices=OpenShiftFacts.known_roles, - required=False), - local_facts=dict(default={}, type='dict', required=False), - ), - supports_check_mode=True, - add_file_common_args=True, + argument_spec=dict( + role=dict(default='common', required=False, + choices=OpenShiftFacts.known_roles), + local_facts=dict(default=None, type='dict', required=False), + ), + supports_check_mode=True, + add_file_common_args=True, ) role = module.params['role'] @@ -464,11 +748,13 @@ def main(): file_params['path'] = fact_file file_args = module.load_file_common_arguments(file_params) changed = module.set_fs_attributes_if_different(file_args, - openshift_facts.changed) + openshift_facts.changed) return module.exit_json(changed=changed, - ansible_facts=openshift_facts.get_facts()) + ansible_facts=openshift_facts.facts) +# ignore pylint errors related to the module_utils import +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import # import module snippets from ansible.module_utils.basic import * from ansible.module_utils.facts import * diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml index 5a7d10d25..d71e6d019 100644 --- a/roles/openshift_facts/tasks/main.yml +++ b/roles/openshift_facts/tasks/main.yml @@ -1,3 +1,9 @@ --- +- name: Verify Ansible version is greater than 1.8.0 and not 1.9.0 + assert: + that: + - ansible_version | version_compare('1.8.0', 'ge') + - ansible_version | version_compare('1.9.0', 'ne') + - name: Gather OpenShift facts openshift_facts: diff --git a/roles/openshift_master/README.md b/roles/openshift_master/README.md index 9f9d0a613..3178e318c 100644 --- a/roles/openshift_master/README.md +++ b/roles/openshift_master/README.md @@ -17,7 +17,7 @@ From this role: |-------------------------------------|-----------------------|--------------------------------------------------| | openshift_master_debug_level | openshift_debug_level | Verbosity of the debug logs for openshift-master | | openshift_node_ips | [] | List of the openshift node ip addresses to pre-register when openshift-master starts up | -| openshift_registry_url | UNDEF | Default docker registry to use | +| oreg_url | UNDEF | Default docker registry to use | | openshift_master_api_port | UNDEF | | | openshift_master_console_port | UNDEF | | | openshift_master_api_url | UNDEF | | diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml index 87fb347a8..11195e83e 100644 --- a/roles/openshift_master/defaults/main.yml +++ b/roles/openshift_master/defaults/main.yml @@ -2,12 +2,19 @@ openshift_node_ips: [] # TODO: update setting these values based on the facts -# TODO: update for console port change os_firewall_allow: - service: etcd embedded port: 4001/tcp - service: OpenShift api https port: 8443/tcp +- service: OpenShift dns tcp + port: 53/tcp +- service: OpenShift dns udp + port: 53/udp +- service: Fluentd td-agent tcp + port: 24224/tcp +- service: Fluentd td-agent udp + port: 24224/udp os_firewall_deny: - service: OpenShift api http port: 8080/tcp diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index f9e6199a5..ac96e2b48 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -49,15 +49,15 @@ # TODO: should probably use a template lookup for this # TODO: should allow for setting --etcd, --kubernetes options # TODO: recreate config if values change -- name: Use enterprise default for openshift_registry_url if not set +- name: Use enterprise default for oreg_url if not set set_fact: - openshift_registry_url: "openshift3_beta/ose-${component}:${version}" - when: openshift.common.deployment_type == 'enterprise' and openshift_registry_url is not defined + oreg_url: "openshift3_beta/ose-${component}:${version}" + when: openshift.common.deployment_type == 'enterprise' and oreg_url is not defined -- name: Use online default for openshift_registry_url if not set +- name: Use online default for oreg_url if not set set_fact: - openshift_registry_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}" - when: openshift.common.deployment_type == 'online' and openshift_registry_url is not defined + oreg_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}" + when: openshift.common.deployment_type == 'online' and oreg_url is not defined - name: Create master config command: > @@ -67,7 +67,7 @@ --master={{ openshift.master.api_url }} --public-master={{ openshift.master.public_api_url }} --listen={{ 'https' if openshift.master.api_use_ssl else 'http' }}://0.0.0.0:{{ openshift.master.api_port }} - {{ ('--images=' ~ openshift_registry_url) if (openshift_registry_url | default('', true) != '') else '' }} + {{ ('--images=' ~ oreg_url) if (oreg_url | default('', true) != '') else '' }} {{ ('--nodes=' ~ openshift_node_ips | join(',')) if (openshift_node_ips | default('', true) != '') else '' }} args: chdir: "{{ openshift_cert_parent_dir }}" diff --git a/roles/openshift_node/README.md b/roles/openshift_node/README.md index 83359f164..c3c17b848 100644 --- a/roles/openshift_node/README.md +++ b/roles/openshift_node/README.md @@ -17,7 +17,7 @@ From this role: | Name | Default value | | |------------------------------------------|-----------------------|----------------------------------------| | openshift_node_debug_level | openshift_debug_level | Verbosity of the debug logs for openshift-node | -| openshift_registry_url | UNDEF (Optional) | Default docker registry to use | +| oreg_url | UNDEF (Optional) | Default docker registry to use | From openshift_common: | Name | Default Value | | diff --git a/roles/openshift_register_nodes/tasks/main.yml b/roles/openshift_register_nodes/tasks/main.yml index d4d72d126..dcb96bbf9 100644 --- a/roles/openshift_register_nodes/tasks/main.yml +++ b/roles/openshift_register_nodes/tasks/main.yml @@ -6,15 +6,15 @@ # TODO: use a template lookup here # TODO: create a failed_when condition -- name: Use enterprise default for openshift_registry_url if not set +- name: Use enterprise default for oreg_url if not set set_fact: - openshift_registry_url: "openshift3_beta/ose-${component}:${version}" - when: openshift.common.deployment_type == 'enterprise' and openshift_registry_url is not defined + oreg_url: "openshift3_beta/ose-${component}:${version}" + when: openshift.common.deployment_type == 'enterprise' and oreg_url is not defined -- name: Use online default for openshift_registry_url if not set +- name: Use online default for oreg_url if not set set_fact: - openshift_registry_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}" - when: openshift.common.deployment_type == 'online' and openshift_registry_url is not defined + oreg_url: "docker-registry.ops.rhcloud.com/openshift3_beta/ose-${component}:${version}" + when: openshift.common.deployment_type == 'online' and oreg_url is not defined - name: Create node config command: > @@ -30,7 +30,7 @@ --certificate-authority={{ openshift_master_ca_cert }} --signer-serial={{ openshift_master_ca_dir }}/serial.txt --node-client-certificate-authority={{ openshift_master_ca_cert }} - {{ ('--images=' ~ openshift_registry_url) if openshift_registry_url is defined else '' }} + {{ ('--images=' ~ oreg_url) if oreg_url is defined else '' }} --listen=https://0.0.0.0:10250 args: chdir: "{{ openshift_cert_parent_dir }}" diff --git a/roles/openshift_registry/README.md b/roles/openshift_registry/README.md new file mode 100644 index 000000000..202c818b8 --- /dev/null +++ b/roles/openshift_registry/README.md @@ -0,0 +1,42 @@ +OpenShift Container Docker Registry +=================================== + +OpenShift Docker Registry service installation + +Requirements +------------ + +Running OpenShift cluster + +Role Variables +-------------- + +From this role: +| Name | Default value | | +|--------------------|-------------------------------------------------------|---------------------| +| | | | + +From openshift_common: +| Name | Default value | | +|-----------------------|---------------|--------------------------------------| +| openshift_debug_level | 0 | Global openshift debug log verbosity | + + +Dependencies +------------ + +Example Playbook +---------------- + +TODO + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +Red Hat openshift@redhat.com + diff --git a/roles/openshift_registry/handlers/main.yml b/roles/openshift_registry/handlers/main.yml new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/roles/openshift_registry/handlers/main.yml diff --git a/roles/openshift_registry/meta/main.yml b/roles/openshift_registry/meta/main.yml new file mode 100644 index 000000000..93b6797d1 --- /dev/null +++ b/roles/openshift_registry/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: OpenShift Red Hat + description: OpenShift Embedded Docker Registry + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.7 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud diff --git a/roles/openshift_registry/tasks/main.yml b/roles/openshift_registry/tasks/main.yml new file mode 100644 index 000000000..7e6982d99 --- /dev/null +++ b/roles/openshift_registry/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- set_fact: _oreg_images="--images={{ oreg_url|quote }}" + when: oreg_url is defined + +- name: Deploy OpenShift Registry + command: openshift admin registry --create --credentials=/var/lib/openshift/openshift.local.certificates/openshift-registry/.kubeconfig {{ _oreg_images|default() }} + register: _oreg_results + changed_when: "'service exists' not in _oreg_results.stdout" diff --git a/roles/openshift_registry/vars/main.yml b/roles/openshift_registry/vars/main.yml new file mode 100644 index 000000000..cd21505a4 --- /dev/null +++ b/roles/openshift_registry/vars/main.yml @@ -0,0 +1,2 @@ +--- + diff --git a/roles/openshift_router/README.md b/roles/openshift_router/README.md new file mode 100644 index 000000000..6d8ee25c6 --- /dev/null +++ b/roles/openshift_router/README.md @@ -0,0 +1,41 @@ +OpenShift Container Router +========================== + +OpenShift Router service installation + +Requirements +------------ + +Running OpenShift cluster + +Role Variables +-------------- + +From this role: +| Name | Default value | | +|--------------------|-------------------------------------------------------|---------------------| +| | | | + +From openshift_common: +| Name | Default value | | +|-----------------------|---------------|--------------------------------------| +| openshift_debug_level | 0 | Global openshift debug log verbosity | + +Dependencies +------------ + +Example Playbook +---------------- + +TODO + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +Red Hat openshift@redhat.com + diff --git a/roles/openshift_router/handlers/main.yml b/roles/openshift_router/handlers/main.yml new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/roles/openshift_router/handlers/main.yml diff --git a/roles/openshift_router/meta/main.yml b/roles/openshift_router/meta/main.yml new file mode 100644 index 000000000..0471e5e14 --- /dev/null +++ b/roles/openshift_router/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: OpenShift Red Hat + description: OpenShift Embedded Router + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.7 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud diff --git a/roles/openshift_router/tasks/main.yml b/roles/openshift_router/tasks/main.yml new file mode 100644 index 000000000..f1ee99dd3 --- /dev/null +++ b/roles/openshift_router/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- set_fact: _ortr_images="--images={{ oreg_url|quote }}" + when: oreg_url is defined + +- name: Deploy OpenShift Router + command: openshift ex router --create --credentials=/var/lib/openshift/openshift.local.certificates/openshift-router/.kubeconfig {{ _ortr_images|default() }} + register: _ortr_results + changed_when: "'service exists' not in _ortr_results.stdout" diff --git a/roles/openshift_router/vars/main.yml b/roles/openshift_router/vars/main.yml new file mode 100644 index 000000000..cd21505a4 --- /dev/null +++ b/roles/openshift_router/vars/main.yml @@ -0,0 +1,2 @@ +--- + diff --git a/roles/os_zabbix/library/zbxapi.py b/roles/os_zabbix/library/zbxapi.py index f4f52909b..b5fa5ee2b 100755 --- a/roles/os_zabbix/library/zbxapi.py +++ b/roles/os_zabbix/library/zbxapi.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +# vim: expandtab:tabstop=4:shiftwidth=4 +''' + ZabbixAPI ansible module +''' # Copyright 2015 Red Hat Inc. # @@ -17,11 +21,22 @@ # Purpose: An ansible module to communicate with zabbix. # +# pylint: disable=line-too-long +# Disabling line length for readability + import json import httplib2 import sys import os import re +import copy + +class ZabbixAPIError(Exception): + ''' + ZabbixAPIError + Exists to propagate errors up from the api + ''' + pass class ZabbixAPI(object): ''' @@ -69,23 +84,26 @@ class ZabbixAPI(object): 'Usermedia': ['get'], } - def __init__(self, data={}): - self.server = data['server'] or None - self.username = data['user'] or None - self.password = data['password'] or None - if any(map(lambda value: value == None, [self.server, self.username, self.password])): + def __init__(self, data=None): + if not data: + data = {} + self.server = data.get('server', None) + self.username = data.get('user', None) + self.password = data.get('password', None) + if any([value == None for value in [self.server, self.username, self.password]]): print 'Please specify zabbix server url, username, and password.' sys.exit(1) - self.verbose = data.has_key('verbose') + self.verbose = data.get('verbose', False) self.use_ssl = data.has_key('use_ssl') self.auth = None - for class_name, method_names in self.classes.items(): - #obj = getattr(self, class_name)(self) - #obj.__dict__ - setattr(self, class_name.lower(), getattr(self, class_name)(self)) + for cname, _ in self.classes.items(): + setattr(self, cname.lower(), getattr(self, cname)(self)) + # pylint: disable=no-member + # This method does not exist until the metaprogramming executed + # This is permanently disabled. results = self.user.login(user=self.username, password=self.password) if results[0]['status'] == '200': @@ -98,48 +116,40 @@ class ZabbixAPI(object): print "Error in call to zabbix. Http status: {0}.".format(results[0]['status']) sys.exit(1) - def perform(self, method, params): + def perform(self, method, rpc_params): ''' This method calls your zabbix server. It requires the following parameters in order for a proper request to be processed: - - jsonrpc - the version of the JSON-RPC protocol used by the API; the Zabbix API implements JSON-RPC version 2.0; + jsonrpc - the version of the JSON-RPC protocol used by the API; + the Zabbix API implements JSON-RPC version 2.0; method - the API method being called; - params - parameters that will be passed to the API method; + rpc_params - parameters that will be passed to the API method; id - an arbitrary identifier of the request; auth - a user authentication token; since we don't have one yet, it's set to null. ''' http_method = "POST" - if params.has_key("http_method"): - http_method = params['http_method'] - jsonrpc = "2.0" - if params.has_key('jsonrpc'): - jsonrpc = params['jsonrpc'] - rid = 1 - if params.has_key('id'): - rid = params['id'] http = None if self.use_ssl: http = httplib2.Http() else: - http = httplib2.Http( disable_ssl_certificate_validation=True,) + http = httplib2.Http(disable_ssl_certificate_validation=True,) - headers = params.get('headers', {}) + headers = {} headers["Content-type"] = "application/json" body = { "jsonrpc": jsonrpc, "method": method, - "params": params, + "params": rpc_params.get('params', {}), "id": rid, 'auth': self.auth, } - if method in ['user.login','api.version']: + if method in ['user.login', 'api.version']: del body['auth'] body = json.dumps(body) @@ -150,48 +160,70 @@ class ZabbixAPI(object): print headers httplib2.debuglevel = 1 - response, results = http.request(self.server, http_method, body, headers) + response, content = http.request(self.server, http_method, body, headers) + + if response['status'] not in ['200', '201']: + raise ZabbixAPIError('Error calling zabbix. Zabbix returned %s' % response['status']) if self.verbose: print response - print results + print content try: - results = json.loads(results) - except ValueError as e: - results = {"error": e.message} + content = json.loads(content) + except ValueError as err: + content = {"error": err.message} - return response, results + return response, content - ''' - This bit of metaprogramming is where the ZabbixAPI subclasses are created. - For each of ZabbixAPI.classes we create a class from the key and methods - from the ZabbixAPI.classes values. We pass a reference to ZabbixAPI class - to each subclass in order for each to be able to call the perform method. - ''' @staticmethod - def meta(class_name, method_names): - # This meta method allows a class to add methods to it. - def meta_method(Class, method_name): + def meta(cname, method_names): + ''' + This bit of metaprogramming is where the ZabbixAPI subclasses are created. + For each of ZabbixAPI.classes we create a class from the key and methods + from the ZabbixAPI.classes values. We pass a reference to ZabbixAPI class + to each subclass in order for each to be able to call the perform method. + ''' + def meta_method(_class, method_name): + ''' + This meta method allows a class to add methods to it. + ''' # This template method is a stub method for each of the subclass # methods. - def template_method(self, **params): - return self.parent.perform(class_name.lower()+"."+method_name, params) - template_method.__doc__ = "https://www.zabbix.com/documentation/2.4/manual/api/reference/%s/%s" % (class_name.lower(), method_name) + def template_method(self, params=None, **rpc_params): + ''' + This template method is a stub method for each of the subclass methods. + ''' + if params: + rpc_params['params'] = params + else: + rpc_params['params'] = copy.deepcopy(rpc_params) + + return self.parent.perform(cname.lower()+"."+method_name, rpc_params) + + template_method.__doc__ = \ + "https://www.zabbix.com/documentation/2.4/manual/api/reference/%s/%s" % \ + (cname.lower(), method_name) template_method.__name__ = method_name # this is where the template method is placed inside of the subclass # e.g. setattr(User, "create", stub_method) - setattr(Class, template_method.__name__, template_method) + setattr(_class, template_method.__name__, template_method) # This class call instantiates a subclass. e.g. User - Class=type(class_name, (object,), { '__doc__': "https://www.zabbix.com/documentation/2.4/manual/api/reference/%s" % class_name.lower() }) - # This init method gets placed inside of the Class - # to allow it to be instantiated. A reference to the parent class(ZabbixAPI) - # is passed in to allow each class access to the perform method. + _class = type(cname, + (object,), + {'__doc__': \ + "https://www.zabbix.com/documentation/2.4/manual/api/reference/%s" % cname.lower()}) def __init__(self, parent): + ''' + This init method gets placed inside of the _class + to allow it to be instantiated. A reference to the parent class(ZabbixAPI) + is passed in to allow each class access to the perform method. + ''' self.parent = parent + # This attaches the init to the subclass. e.g. Create - setattr(Class, __init__.__name__, __init__) + setattr(_class, __init__.__name__, __init__) # For each of our ZabbixAPI.classes dict values # Create a method and attach it to our subclass. # e.g. 'User': ['delete', 'get', 'updatemedia', 'updateprofile', @@ -200,25 +232,54 @@ class ZabbixAPI(object): # User.delete # User.get for method_name in method_names: - meta_method(Class, method_name) + meta_method(_class, method_name) # Return our subclass with all methods attached - return Class + return _class # Attach all ZabbixAPI.classes to ZabbixAPI class through metaprogramming -for class_name, method_names in ZabbixAPI.classes.items(): - setattr(ZabbixAPI, class_name, ZabbixAPI.meta(class_name, method_names)) +for _class_name, _method_names in ZabbixAPI.classes.items(): + setattr(ZabbixAPI, _class_name, ZabbixAPI.meta(_class_name, _method_names)) + +def exists(content, key='result'): + ''' Check if key exists in content or the size of content[key] > 0 + ''' + if not content.has_key(key): + return False + + if not content[key]: + return False + + return True + +def diff_content(from_zabbix, from_user): + ''' Compare passed in object to results returned from zabbix + ''' + terms = ['search', 'output', 'groups', 'select', 'expand'] + regex = '(' + '|'.join(terms) + ')' + retval = {} + for key, value in from_user.items(): + if re.findall(regex, key): + continue + + if from_zabbix[key] != str(value): + retval[key] = str(value) + + return retval def main(): + ''' + This main method runs the ZabbixAPI Ansible Module + ''' module = AnsibleModule( - argument_spec = dict( + argument_spec=dict( server=dict(default='https://localhost/zabbix/api_jsonrpc.php', type='str'), user=dict(default=None, type='str'), password=dict(default=None, type='str'), zbx_class=dict(choices=ZabbixAPI.classes.keys()), - action=dict(default=None, type='str'), params=dict(), debug=dict(default=False, type='bool'), + state=dict(default='present', type='str'), ), #supports_check_mode=True ) @@ -227,47 +288,83 @@ def main(): if not user: user = os.environ['ZABBIX_USER'] - pw = module.params.get('password', None) - if not pw: - pw = os.environ['ZABBIX_PASSWORD'] + passwd = module.params.get('password', None) + if not passwd: + passwd = os.environ['ZABBIX_PASSWORD'] - server = module.params['server'] - if module.params['debug']: - options['debug'] = True api_data = { 'user': user, - 'password': pw, - 'server': server, + 'password': passwd, + 'server': module.params['server'], + 'verbose': module.params['debug'] } - if not user or not pw or not server: - module.fail_json('Please specify the user, password, and the zabbix server.') + if not user or not passwd or not module.params['server']: + module.fail_json(msg='Please specify the user, password, and the zabbix server.') zapi = ZabbixAPI(api_data) zbx_class = module.params.get('zbx_class') - action = module.params.get('action') - params = module.params.get('params', {}) - + rpc_params = module.params.get('params', {}) + state = module.params.get('state') # Get the instance we are trying to call zbx_class_inst = zapi.__getattribute__(zbx_class.lower()) - # Get the instance's method we are trying to call - zbx_action_method = zapi.__getattribute__(zbx_class.capitalize()).__dict__[action] - # Make the call with the incoming params - results = zbx_action_method(zbx_class_inst, **params) - - # Results Section - changed_state = False - status = results[0]['status'] - if status not in ['200', '201']: - #changed_state = False - module.fail_json(msg="Http response: [%s] - Error: %s" % (str(results[0]), results[1])) - module.exit_json(**{'results': results[1]['result']}) + # perform get + # Get the instance's method we are trying to call + zbx_action_method = zapi.__getattribute__(zbx_class.capitalize()).__dict__['get'] + _, content = zbx_action_method(zbx_class_inst, rpc_params) + + if state == 'list': + module.exit_json(changed=False, results=content['result'], state="list") + + if state == 'absent': + if not exists(content): + module.exit_json(changed=False, state="absent") + # If we are coming from a query, we need to pass in the correct rpc_params for delete. + # specifically the zabbix class name + 'id' + # if rpc_params is a list then we need to pass it. (list of ids to delete) + idname = zbx_class.lower() + "id" + if not isinstance(rpc_params, list) and content['result'][0].has_key(idname): + rpc_params = [content['result'][0][idname]] + + zbx_action_method = zapi.__getattribute__(zbx_class.capitalize()).__dict__['delete'] + _, content = zbx_action_method(zbx_class_inst, rpc_params) + module.exit_json(changed=True, results=content['result'], state="absent") + + if state == 'present': + # It's not there, create it! + if not exists(content): + zbx_action_method = zapi.__getattribute__(zbx_class.capitalize()).__dict__['create'] + _, content = zbx_action_method(zbx_class_inst, rpc_params) + module.exit_json(changed=True, results=content['result'], state='present') + + # It's there and the same, do nothing! + diff_params = diff_content(content['result'][0], rpc_params) + if not diff_params: + module.exit_json(changed=False, results=content['result'], state="present") + + # Add the id to update with + idname = zbx_class.lower() + "id" + diff_params[idname] = content['result'][0][idname] + + + ## It's there and not the same, update it! + zbx_action_method = zapi.__getattribute__(zbx_class.capitalize()).__dict__['update'] + _, content = zbx_action_method(zbx_class_inst, diff_params) + module.exit_json(changed=True, results=content, state="present") + + module.exit_json(failed=True, + changed=False, + results='Unknown state passed. %s' % state, + state="unknown") + +# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import, locally-disabled +# import module snippets. This are required from ansible.module_utils.basic import * main() |