cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

foreman.py: create Ansible groups from Satellite 6 host collections (#25269)

Browse source 
* create Ansible groups from host collections

* fix paging logic in _get_json

* replace Satellite with Foreman

* improve comment for environment variables

* enable host collections by default

* use requests params instead of urllib.quote_plus

* disable host collections by default

* change organization filter

* clean up load_cache methods

* remove usage of function defaults

* replace environment variables with host_filters
This commit is contained in:
Carsten Clasohm 2017-06-09 17:10:55 -07:00 committed by Matt Davis
parent 392934f1b9
commit cd31f4a102
2 changed files with 99 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
contrib/inventory/foreman.ini
View file

@ -68,6 +68,21 @@
#
# foreman_hostgroup_myapp_webtier_datacenter1
#
# If the parameter want_hostcollections is set to true, the
# collections each host is in are created as Ansible groups with a
# foreman_hostcollection prefix, all lowercase and problematic
# parameters removed. So e.g. the Foreman host collection
#
# Patch Window Thursday
#
# would turn into the Ansible group:
#
# foreman_hostcollection_patchwindowthursday
#
# If the parameter host_filters is set, it will be used as the
# "search" parameter for the /api/v2/hosts call. This can be used to
# restrict the list of returned host, as shown below.
#
# Furthermore Ansible groups can be created on the fly using the
# *group_patterns* variable in *foreman.ini* so that you can build up
# hierarchies using parameters on the hostgroup and host variables.
@ -108,15 +123,28 @@ user = foreman
password = secret
ssl_verify = True
# Retrieve only hosts from the organization "Web Engineering".
# host_filters = organization="Web Engineering"
# Retrieve only hosts from the organization "Web Engineering" that are
# also in the host collection "Apache Servers".
# host_filters = organization="Web Engineering" and host_collection="Apache Servers"
[ansible]
group_patterns = ["{app}-{tier}-{color}",
"{app}-{color}",
"{app}",
"{tier}"]
group_prefix = foreman_
# Whether to fetch facts from Foreman and store them on the host
want_facts = True
# Whether to create Ansible groups for host collections. Only tested
# with Katello (Red Hat Satellite). Disabled by default to not break
# the script for stand-alone Foreman.
want_hostcollections = False
[cache]
path = .
max_age = 60

96
contrib/inventory/foreman.py
View file

@ -64,6 +64,7 @@ class ForemanInventory(object):
self.params = dict() # Params of each host
self.facts = dict() # Facts of each host
self.hostgroups = dict() # host groups
self.hostcollections = dict() # host collections
self.session = None # Requests session
self.config_paths = [
"/etc/ansible/foreman.ini",
@ -107,6 +108,16 @@ class ForemanInventory(object):
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
self.want_facts = True
try:
self.want_hostcollections = config.getboolean('ansible', 'want_hostcollections')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
self.want_hostcollections = False
try:
self.host_filters = config.get('foreman', 'host_filters')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
self.host_filters = None
# Cache related
try:
cache_path = os.path.expanduser(config.get('cache', 'path'))
@ -117,10 +128,12 @@ class ForemanInventory(object):
self.cache_path_inventory = cache_path + "/%s.index" % script
self.cache_path_params = cache_path + "/%s.params" % script
self.cache_path_facts = cache_path + "/%s.facts" % script
self.cache_path_hostcollections = cache_path + "/%s.hostcollections" % script
try:
self.cache_max_age = config.getint('cache', 'max_age')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
self.cache_max_age = 60
return True
def parse_cli_args(self):
@ -140,12 +153,17 @@ class ForemanInventory(object):
self.session.verify = self.foreman_ssl_verify
return self.session
def _get_json(self, url, ignore_errors=None):
def _get_json(self, url, ignore_errors=None, params=None):
if params is None:
params = {}
params['per_page'] = 250
page = 1
results = []
s = self._get_session()
while True:
ret = s.get(url, params={'page': page, 'per_page': 250})
params['page'] = page
ret = s.get(url, params=params)
if ignore_errors and ret.status_code in ignore_errors:
break
ret.raise_for_status()
@ -158,7 +176,7 @@ class ForemanInventory(object):
return json['results']
# List of all hosts is returned paginaged
results = results + json['results']
if len(results) >= json['total']:
if len(results) >= json['subtotal']:
break
page += 1
if len(json['results']) == 0:
@ -169,20 +187,27 @@ class ForemanInventory(object):
return results
def _get_hosts(self):
return self._get_json("%s/api/v2/hosts" % self.foreman_url)
url = "%s/api/v2/hosts" % self.foreman_url
def _get_all_params_by_id(self, hid):
params = {}
if self.host_filters:
params['search'] = self.host_filters
return self._get_json(url, params=params)
def _get_host_data_by_id(self, hid):
url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid)
ret = self._get_json(url, [404])
if ret == []:
ret = {}
return ret.get('all_parameters', {})
return self._get_json(url)
def _resolve_params(self, host):
"""Fetch host params and convert to dict"""
def _get_facts_by_id(self, hid):
url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid)
return self._get_json(url)
def _resolve_params(self, host_params):
"""Convert host params to dict"""
params = {}
for param in self._get_all_params_by_id(host['id']):
for param in host_params:
name = param['name']
params[name] = param['value']
@ -218,6 +243,7 @@ class ForemanInventory(object):
self.write_to_cache(self.inventory, self.cache_path_inventory)
self.write_to_cache(self.params, self.cache_path_params)
self.write_to_cache(self.facts, self.cache_path_facts)
self.write_to_cache(self.hostcollections, self.cache_path_hostcollections)
def to_safe(self, word):
'''Converts 'bad' characters in a string to underscores
@ -238,6 +264,9 @@ class ForemanInventory(object):
for host in self._get_hosts():
dns_name = host['name']
host_data = self._get_host_data_by_id(host['id'])
host_params = host_data.get('all_parameters', {})
# Create ansible groups for hostgroup
group = 'hostgroup'
val = host.get('%s_title' % group) or host.get('%s_name' % group)
@ -258,7 +287,7 @@ class ForemanInventory(object):
safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
self.inventory[safe_key].append(dns_name)
params = self._resolve_params(host)
params = self._resolve_params(host_params)
# Ansible groups by parameters in host groups and Foreman host
# attributes.
@ -274,6 +303,17 @@ class ForemanInventory(object):
except KeyError:
pass # Host not part of this group
if self.want_hostcollections:
hostcollections = host_data.get('host_collections')
if hostcollections:
# Create Ansible groups for host collections
for hostcollection in hostcollections:
safe_key = self.to_safe('%shostcollection_%s' % (self.group_prefix, hostcollection['name'].lower()))
self.inventory[safe_key].append(dns_name)
self.hostcollections[dns_name] = hostcollections
self.cache[dns_name] = host
self.params[dns_name] = params
self.facts[dns_name] = self._get_facts(host)
@ -295,31 +335,36 @@ class ForemanInventory(object):
def load_inventory_from_cache(self):
"""Read the index from the cache file sets self.index"""
cache = open(self.cache_path_inventory, 'r')
json_inventory = cache.read()
self.inventory = json.loads(json_inventory)
with open(self.cache_path_inventory, 'r') as fp:
self.inventory = json.load(fp)
def load_params_from_cache(self):
"""Read the index from the cache file sets self.index"""
cache = open(self.cache_path_params, 'r')
json_params = cache.read()
self.params = json.loads(json_params)
with open(self.cache_path_params, 'r') as fp:
self.params = json.load(fp)
def load_facts_from_cache(self):
"""Read the index from the cache file sets self.facts"""
if not self.want_facts:
return
cache = open(self.cache_path_facts, 'r')
json_facts = cache.read()
self.facts = json.loads(json_facts)
with open(self.cache_path_facts, 'r') as fp:
self.facts = json.load(fp)
def load_hostcollections_from_cache(self):
"""Read the index from the cache file sets self.hostcollections"""
if not self.want_hostcollections:
return
with open(self.cache_path_hostcollections, 'r') as fp:
self.hostcollections = json.load(fp)
def load_cache_from_cache(self):
"""Read the cache from the cache file sets self.cache"""
cache = open(self.cache_path_cache, 'r')
json_cache = cache.read()
self.cache = json.loads(json_cache)
with open(self.cache_path_cache, 'r') as fp:
self.cache = json.load(fp)
def get_inventory(self):
if self.args.refresh_cache or not self.is_cache_valid():
@ -328,6 +373,7 @@ class ForemanInventory(object):
self.load_inventory_from_cache()
self.load_params_from_cache()
self.load_facts_from_cache()
self.load_hostcollections_from_cache()
self.load_cache_from_cache()
def get_host_info(self):