cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[GCE] Caching support for inventory script. (#18093)

Browse source 
* [GCE] Caching support for inventory script.

The GCE inventory script now supports reading from a cache rather than making the request each time.  The format of the list and host output have not changed.

On script execution, the cache is checked to see if it older than 'cache_max_age', and if so, it is rebuilt (it can also be explicity rebuilt).

To support this functionality, the following have been added.

* Config file (gce.ini) changes: A new 'cache' section has been added to the config file, with 'cache_path' and 'cache_max_age' options to allow for configuration.  There are intelligent defaults in place if that section and options are not found in the configuration file.

* Command line argument: A new --refresh-cache argument has been added to force the cache to be rebuild.

* A CloudInventoryCache class, contained in the same file has been added.  As a seperate class, it allowed for testing (unit tests not included in this PR) and hopefully could be re-used in the future (it contains borrowed code from other inventory scripts)

* load_inventory_from_cache and do_api_calls_and_update_cache methods (, which were largely lifted from other inventory scripts, in a hope to promote consistency in the future) to determine if the cache is fresh and rebuild if necessary.

* A 'main' check, to support the script being imported and testable.

A new dictionary has been added to the list output, located at ['_meta']['stats'] that informs if the cache was used and how long it took to load the inventory (in 'cache_used' and 'inventory_load_time', respectively).

* fixed default value error; change cache time to 300
This commit is contained in:
Tom Melendez 2016-10-24 13:32:50 -07:00 committed by Ryan Brown
parent 460da3b537
commit 54caf3c5d5
2 changed files with 120 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
contrib/inventory/gce.ini
View file

@ -37,6 +37,7 @@
# exist in your PYTHONPATH and be picked up automatically with an import
# statement in the inventory script. However, you can specify an absolute
# path to the secrets.py file with 'libcloud_secrets' parameter.
# This option will be deprecated in a future release.
libcloud_secrets =
# If you are not going to use a 'secrets.py' file, you can set the necessary
@ -58,3 +59,11 @@ gce_project_id =
# The INVENTORY_IP_TYPE environment variable will override this value.
inventory_ip_type =
[cache]
# directory in which cache should be created
cache_path = ~/.ansible/tmp
# The number of seconds a cache file is considered valid. After this many
# seconds, a new API call will be made, and the cache file will be updated.
# To disable the cache, set this value to 0
cache_max_age = 300

123
contrib/inventory/gce.py
View file

@ -69,8 +69,8 @@ Examples:
$ contrib/inventory/gce.py --host my_instance
Author: Eric Johnson <erjohnso@google.com>
Contributors: Matt Hite <mhite@hotmail.com>
Version: 0.0.2
Contributors: Matt Hite <mhite@hotmail.com>, Tom Melendez <supertom@google.com>
Version: 0.0.3
'''
__requires__ = ['pycrypto>=2.6']
@ -89,6 +89,9 @@ USER_AGENT_VERSION="v2"
import sys
import os
import argparse
from time import time
import ConfigParser
import logging
@ -107,8 +110,57 @@ except:
sys.exit("GCE inventory script requires libcloud >= 0.13")
class CloudInventoryCache(object):
def __init__(self, cache_name='ansible-cloud-cache', cache_path='/tmp',
cache_max_age=300):
cache_dir = os.path.expanduser(cache_path)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
self.cache_path_cache = os.path.join(cache_dir, cache_name)
self.cache_max_age = cache_max_age
def is_valid(self, max_age=None):
''' Determines if the cache files have expired, or if it is still valid '''
if max_age is None:
max_age = self.cache_max_age
if os.path.isfile(self.cache_path_cache):
mod_time = os.path.getmtime(self.cache_path_cache)
current_time = time()
if (mod_time + max_age) > current_time:
return True
return False
def get_all_data_from_cache(self, filename=''):
''' Reads the JSON inventory from the cache file. Returns Python dictionary. '''
data = ''
if not filename:
filename = self.cache_path_cache
with open(filename, 'r') as cache:
data = cache.read()
return json.loads(data)
def write_to_cache(self, data, filename=''):
''' Writes data to file as JSON. Returns True. '''
if not filename:
filename = self.cache_path_cache
json_data = json.dumps(data)
with open(filename, 'w') as cache:
cache.write(json_data)
return True
class GceInventory(object):
def __init__(self):
# Cache object
self.cache = None
# dictionary containing inventory read from disk
self.inventory = {}
# Read settings and parse CLI arguments
self.parse_cli_args()
self.config = self.get_config()
@ -117,22 +169,36 @@ class GceInventory(object):
if self.ip_type:
self.ip_type = self.ip_type.lower()
# Cache management
start_inventory_time = time()
cache_used = False
if self.args.refresh_cache or not self.cache.is_valid():
self.do_api_calls_update_cache()
else:
self.load_inventory_from_cache()
cache_used = True
self.inventory['_meta']['stats'] = {'use_cache': True}
self.inventory['_meta']['stats'] = {
'inventory_load_time': time() - start_inventory_time,
'cache_used': cache_used
}
# Just display data for specific host
if self.args.host:
print(self.json_format_dict(self.node_to_dict(
self.get_instance(self.args.host)),
print(self.json_format_dict(
self.inventory['_meta']['hostvars'][self.args.host],
pretty=self.args.pretty))
sys.exit(0)
zones = self.parse_env_zones()
else:
# Otherwise, assume user wants all instances grouped
print(self.json_format_dict(self.group_instances(zones),
zones = self.parse_env_zones()
print(self.json_format_dict(self.inventory,
pretty=self.args.pretty))
sys.exit(0)
def get_config(self):
"""
Reads the settings from the gce.ini file.
Populates a SafeConfigParser object with defaults and
attempts to read an .ini-style configuration from the filename
specified in GCE_INI_PATH. If the environment variable is
@ -153,11 +219,15 @@ class GceInventory(object):
'gce_project_id': '',
'libcloud_secrets': '',
'inventory_ip_type': '',
'cache_path': '~/.ansible/tmp',
'cache_max_age': '300'
})
if 'gce' not in config.sections():
config.add_section('gce')
if 'inventory' not in config.sections():
config.add_section('inventory')
if 'cache' not in config.sections():
config.add_section('cache')
config.read(gce_ini_path)
@ -173,6 +243,14 @@ class GceInventory(object):
if states:
self.instance_states = states.split(',')
# Caching
cache_path = config.get('cache', 'cache_path')
cache_max_age = config.getint('cache', 'cache_max_age')
# TOOD(supertom): support project-specific caches
cache_name = 'ansible-gce.cache'
self.cache = CloudInventoryCache(cache_path=cache_path,
cache_max_age=cache_max_age,
cache_name=cache_name)
return config
def get_inventory_options(self):
@ -252,6 +330,9 @@ class GceInventory(object):
help='Get all information about an instance')
parser.add_argument('--pretty', action='store_true', default=False,
help='Pretty format (default: False)')
parser.add_argument(
'--refresh-cache', action='store_true', default=False,
help='Force refresh of cache by making API requests (default: False - use cache files)')
self.args = parser.parse_args()
@ -290,12 +371,24 @@ class GceInventory(object):
'ansible_ssh_host': ssh_host
}
def get_instance(self, instance_name):
'''Gets details about a specific instance '''
def load_inventory_from_cache(self):
''' Loads inventory from JSON on disk. '''
try:
return self.driver.ex_get_node(instance_name)
self.inventory = self.cache.get_all_data_from_cache()
hosts = self.inventory['_meta']['hostvars']
except Exception as e:
return None
print(
"Invalid inventory file %s. Please rebuild with -refresh-cache option."
% (self.cache.cache_path_cache))
raise
def do_api_calls_update_cache(self):
''' Do API calls and save data in cache. '''
zones = self.parse_env_zones()
data = self.group_instances(zones)
self.cache.write_to_cache(data)
self.inventory = data
def group_instances(self, zones=None):
'''Group all instances'''
@ -369,6 +462,6 @@ class GceInventory(object):
else:
return json.dumps(data)
# Run the script
GceInventory()
if __name__ == '__main__':
GceInventory()