From b54e8f81469807cdd6cc57d3c03b22ee1212b4cc Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Thu, 7 May 2015 11:09:00 -0400 Subject: Allow option in multi_ec2 to set cache location. --- inventory/multi_ec2.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'inventory/multi_ec2.py') diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index b839a33ea..b99212dd5 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -11,9 +11,12 @@ import yaml import os import subprocess import json +import errno +import fcntl CONFIG_FILE_NAME = 'multi_ec2.yaml' +DEFAULT_CACHE_PATH = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') class MultiEc2(object): ''' @@ -27,7 +30,6 @@ class MultiEc2(object): self.config = None self.all_ec2_results = {} self.result = {} - self.cache_path = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) same_dir_config_file = os.path.join(self.file_path, CONFIG_FILE_NAME) @@ -48,10 +50,12 @@ class MultiEc2(object): self.config = self.load_yaml_config() elif os.environ.has_key("AWS_ACCESS_KEY_ID") and \ os.environ.has_key("AWS_SECRET_ACCESS_KEY"): + # Build a default config self.config = {} self.config['accounts'] = [ { 'name': 'default', + 'cache_location': DEFAULT_CACHE_PATH, 'provider': 'aws/hosts/ec2.py', 'env_vars': { 'AWS_ACCESS_KEY_ID': os.environ["AWS_ACCESS_KEY_ID"], @@ -64,6 +68,11 @@ class MultiEc2(object): else: raise RuntimeError("Could not find valid ec2 credentials in the environment.") + # Set the default cache path but if its defined we'll assign it. + self.cache_path = DEFAULT_CACHE_PATH + if self.config.has_key('cache_location'): + self.cache_path = self.config['cache_location'] + if self.args.refresh_cache: self.get_inventory() self.write_to_cache() @@ -222,9 +231,22 @@ class MultiEc2(object): def write_to_cache(self): ''' Writes data in JSON format to a file ''' + # if it does not exist, try and create it. + if not os.path.isfile(self.cache_path): + path = os.path.dirname(self.cache_path) + try: + os.makedirs(path) + except OSError as exc: + if exc.errno != errno.EEXIST or not os.path.isdir(path): + raise + json_data = MultiEc2.json_format_dict(self.result, True) with open(self.cache_path, 'w') as cache: - cache.write(json_data) + try: + fcntl.flock(cache, fcntl.LOCK_EX) + cache.write(json_data) + finally: + fcntl.flock(cache, fcntl.LOCK_UN) def get_inventory_from_cache(self): ''' Reads the inventory from the cache file and returns it as a JSON -- cgit v1.2.3 From c6c0463a54af4bdab6810697a1ab5f81ef782f4d Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Thu, 7 May 2015 17:36:08 -0400 Subject: Fixed a bug due to renaming of variables. --- inventory/multi_ec2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'inventory/multi_ec2.py') diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index b839a33ea..35ac6c7a8 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -182,7 +182,7 @@ class MultiEc2(object): elif isinstance(input_a[key], list) and isinstance(input_b[key], list): for result in input_b[key]: if result not in input_a[key]: - input_a[key].input_append(result) + input_a[key].append(result) # a is a list and not b elif isinstance(input_a[key], list): if input_b[key] not in input_a[key]: -- cgit v1.2.3 From 7c905c0cf962ec9b7f5bd140a506bd614831f0e8 Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Tue, 12 May 2015 12:20:11 -0400 Subject: Added capability to pass in ec2.ini file. --- inventory/multi_ec2.py | 74 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 21 deletions(-) (limited to 'inventory/multi_ec2.py') diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index 063a80300..d251c6a6a 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -13,7 +13,7 @@ import subprocess import json import errno import fcntl - +import tempfile CONFIG_FILE_NAME = 'multi_ec2.yaml' DEFAULT_CACHE_PATH = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') @@ -128,6 +128,54 @@ class MultiEc2(object): return subprocess.Popen(cmds, stderr=subprocess.PIPE, \ stdout=subprocess.PIPE, env=env) + + @staticmethod + def generate_config(config_data): + """Generate the ec2.ini file in as a secure temp file. + Once generated, pass it to the ec2.py as an environment variable. + """ + fildes, tmp_file_path = tempfile.mkstemp(prefix='multi_ec2.ini.') + for section, values in config_data.items(): + os.write(fildes, "[%s]\n" % section) + for option, value in values.items(): + os.write(fildes, "%s = %s\n" % (option, value)) + os.close(fildes) + return tmp_file_path + + def run_provider(self): + '''Setup the provider call with proper variables + and call self.get_provider_tags. + ''' + try: + all_results = [] + tmp_file_path = None + processes = {} + for account in self.config['accounts']: + env = account['env_vars'] + if account.has_key('provider_config'): + tmp_file_path = MultiEc2.generate_config(account['provider_config']) + env['EC2_INI_PATH'] = tmp_file_path + name = account['name'] + provider = account['provider'] + processes[name] = self.get_provider_tags(provider, env) + + # for each process collect stdout when its available + for name, process in processes.items(): + out, err = process.communicate() + all_results.append({ + "name": name, + "out": out.strip(), + "err": err.strip(), + "code": process.returncode + }) + + finally: + # Clean up the mkstemp file + if tmp_file_path: + os.unlink(tmp_file_path) + + return all_results + def get_inventory(self): """Create the subprocess to fetch tags from a provider. Host query: @@ -138,28 +186,12 @@ class MultiEc2(object): Query all of the different accounts for their tags. Once completed store all of their results into one merged updated hash. """ - processes = {} - for account in self.config['accounts']: - env = account['env_vars'] - name = account['name'] - provider = account['provider'] - processes[name] = self.get_provider_tags(provider, env) - - # for each process collect stdout when its available - all_results = [] - for name, process in processes.items(): - out, err = process.communicate() - all_results.append({ - "name": name, - "out": out.strip(), - "err": err.strip(), - "code": process.returncode - }) + provider_results = self.run_provider() # process --host results if not self.args.host: # For any non-zero, raise an error on it - for result in all_results: + for result in provider_results: if result['code'] != 0: raise RuntimeError(result['err']) else: @@ -171,9 +203,9 @@ class MultiEc2(object): else: # For any 0 result, return it count = 0 - for results in all_results: + for results in provider_results: if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': - self.result = json.loads(out) + self.result = json.loads(results['out']) count += 1 if count > 1: raise RuntimeError("Found > 1 results for --host %s. \ -- cgit v1.2.3 From d82c71ce9a98c1e9ecabf24cd7bd7c7e19aabec2 Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Wed, 13 May 2015 16:38:20 -0400 Subject: Added support for grouping and a bug fix. --- inventory/multi_ec2.py | 64 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 16 deletions(-) (limited to 'inventory/multi_ec2.py') diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index d251c6a6a..11247f942 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -14,6 +14,7 @@ import json import errno import fcntl import tempfile +import copy CONFIG_FILE_NAME = 'multi_ec2.yaml' DEFAULT_CACHE_PATH = os.path.expanduser('~/.ansible/tmp/multi_ec2_inventory.cache') @@ -148,13 +149,13 @@ class MultiEc2(object): ''' try: all_results = [] - tmp_file_path = None + tmp_file_paths = [] processes = {} for account in self.config['accounts']: env = account['env_vars'] if account.has_key('provider_config'): - tmp_file_path = MultiEc2.generate_config(account['provider_config']) - env['EC2_INI_PATH'] = tmp_file_path + tmp_file_paths.append(MultiEc2.generate_config(account['provider_config'])) + env['EC2_INI_PATH'] = tmp_file_paths[-1] name = account['name'] provider = account['provider'] processes[name] = self.get_provider_tags(provider, env) @@ -171,8 +172,8 @@ class MultiEc2(object): finally: # Clean up the mkstemp file - if tmp_file_path: - os.unlink(tmp_file_path) + for tmp_file in tmp_file_paths: + os.unlink(tmp_file) return all_results @@ -189,27 +190,58 @@ class MultiEc2(object): provider_results = self.run_provider() # process --host results - if not self.args.host: + # For any 0 result, return it + if self.args.host: + count = 0 + for results in provider_results: + if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': + self.result = json.loads(results['out']) + count += 1 + if count > 1: + raise RuntimeError("Found > 1 results for --host %s. \ + This is an invalid state." % self.args.host) + # process --list results + else: # For any non-zero, raise an error on it for result in provider_results: if result['code'] != 0: raise RuntimeError(result['err']) else: self.all_ec2_results[result['name']] = json.loads(result['out']) + + # Check if user wants extra vars in yaml by + # having hostvars and all_group defined + for acc_config in self.config['accounts']: + self.apply_account_config(acc_config) + + # Build results by merging all dictionaries values = self.all_ec2_results.values() values.insert(0, self.result) for result in values: MultiEc2.merge_destructively(self.result, result) - else: - # For any 0 result, return it - count = 0 - for results in provider_results: - if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': - self.result = json.loads(results['out']) - count += 1 - if count > 1: - raise RuntimeError("Found > 1 results for --host %s. \ - This is an invalid state." % self.args.host) + + def apply_account_config(self, acc_config): + ''' Apply account config settings + ''' + if not acc_config.has_key('hostvars') and not acc_config.has_key('all_group'): + return + + results = self.all_ec2_results[acc_config['name']] + # Update each hostvar with the newly desired key: value + for host_property, value in acc_config['hostvars'].items(): + # Verify the account results look sane + # by checking for these keys ('_meta' and 'hostvars' exist) + if results.has_key('_meta') and results['_meta'].has_key('hostvars'): + for data in results['_meta']['hostvars'].values(): + data[str(host_property)] = str(value) + + # Add this group + results["%s_%s" % (host_property, value)] = \ + copy.copy(results[acc_config['all_group']]) + + # store the results back into all_ec2_results + self.all_ec2_results[acc_config['name']] = results + @staticmethod def merge_destructively(input_a, input_b): "merges b into input_a" -- cgit v1.2.3 From 2657d56660495476d4e64bf4b1a47ebf277770ee Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Thu, 14 May 2015 12:14:27 -0400 Subject: Making multi_ec2 into a library --- inventory/multi_ec2.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'inventory/multi_ec2.py') diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py index 11247f942..f8196aefd 100755 --- a/inventory/multi_ec2.py +++ b/inventory/multi_ec2.py @@ -26,8 +26,14 @@ class MultiEc2(object): Stores a json hash of resources in result. ''' - def __init__(self): - self.args = None + def __init__(self, args=None): + # Allow args to be passed when called as a library + if not args: + self.args = {} + else: + self.args = args + + self.cache_path = DEFAULT_CACHE_PATH self.config = None self.all_ec2_results = {} self.result = {} @@ -44,8 +50,15 @@ class MultiEc2(object): else: self.config_file = None # expect env vars - self.parse_cli_args() + def run(self): + '''This method checks to see if the local + cache is valid for the inventory. + + if the cache is valid; return cache + else the credentials are loaded from multi_ec2.yaml or from the env + and we attempt to get the inventory from the provider specified. + ''' # load yaml if self.config_file and os.path.isfile(self.config_file): self.config = self.load_yaml_config() @@ -70,15 +83,14 @@ class MultiEc2(object): raise RuntimeError("Could not find valid ec2 credentials in the environment.") # Set the default cache path but if its defined we'll assign it. - self.cache_path = DEFAULT_CACHE_PATH if self.config.has_key('cache_location'): self.cache_path = self.config['cache_location'] - if self.args.refresh_cache: + if self.args.get('refresh_cache', None): self.get_inventory() self.write_to_cache() # if its a host query, fetch and do not cache - elif self.args.host: + elif self.args.get('host', None): self.get_inventory() elif not self.is_cache_valid(): # go fetch the inventories and cache them if cache is expired @@ -119,9 +131,9 @@ class MultiEc2(object): "and that it is executable. (%s)" % provider) cmds = [provider] - if self.args.host: + if self.args.get('host', None): cmds.append("--host") - cmds.append(self.args.host) + cmds.append(self.args.get('host', None)) else: cmds.append('--list') @@ -191,7 +203,7 @@ class MultiEc2(object): # process --host results # For any 0 result, return it - if self.args.host: + if self.args.get('host', None): count = 0 for results in provider_results: if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': @@ -199,7 +211,7 @@ class MultiEc2(object): count += 1 if count > 1: raise RuntimeError("Found > 1 results for --host %s. \ - This is an invalid state." % self.args.host) + This is an invalid state." % self.args.get('host', None)) # process --list results else: # For any non-zero, raise an error on it @@ -290,7 +302,7 @@ class MultiEc2(object): help='List instances (default: True)') parser.add_argument('--host', action='store', default=False, help='Get all the variables about a specific instance') - self.args = parser.parse_args() + self.args = parser.parse_args().__dict__ def write_to_cache(self): ''' Writes data in JSON format to a file ''' @@ -340,4 +352,7 @@ class MultiEc2(object): if __name__ == "__main__": - print MultiEc2().result_str() + MEC2 = MultiEc2() + MEC2.parse_cli_args() + MEC2.run() + print MEC2.result_str() -- cgit v1.2.3