From 1942cd33dc3506b13b89bed871e43f7df7344a0d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 7 Apr 2016 16:22:36 -0400 Subject: [PATCH] draft add group merge priority and yaml inventory * now you can specify a yaml invenotry file * ansible_group_priority will now set this property on groups * added example yaml inventory * TODO: make group var merging depend on priority groups, child/parent relationships should remain unchanged. --- examples/hosts.yaml | 43 +++++++ lib/ansible/inventory/__init__.py | 9 +- lib/ansible/inventory/dir.py | 20 ++- lib/ansible/inventory/group.py | 9 +- lib/ansible/inventory/ini.py | 5 +- lib/ansible/inventory/yaml.py | 200 ++++++++++++++++++++++++++++++ 6 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 examples/hosts.yaml create mode 100644 lib/ansible/inventory/yaml.py diff --git a/examples/hosts.yaml b/examples/hosts.yaml new file mode 100644 index 00000000000..8a2f6b25676 --- /dev/null +++ b/examples/hosts.yaml @@ -0,0 +1,43 @@ +# This is the default ansible 'hosts' file. +# +# It should live in /etc/ansible/hosts +# +# - Comments begin with the '#' character +# - Blank lines are ignored +# - Top level entries are assumed to be groups +# - Hosts must be specified in a group's hosts: +# and they must be a key (: terminated) +# - groups can have children, hosts and vars keys +# - Anything defined under a hosts is assumed to be a var +# - You can enter hostnames or ip addresses +# - A hostname/ip can be a member of multiple groups +# Ex 1: Ungrouped hosts, put in 'ungrouped' group +##ungrouped: +## hosts: +## green.example.com: +## ansible_ssh_host: 191.168.100.32 +## blue.example.com: +## 192.168.100.1: +## 192.168.100.10: + +# Ex 2: A collection of hosts belonging to the 'webservers' group + +##webservers: +## hosts: +## alpha.example.org: +## beta.example.org: +## 192.168.1.100: +## 192.168.1.110: + +# Ex 3: You can create hosts using ranges and add children groups and vars to a group +# The child group can define anything you would normall add to a group + +##testing: +## hosts: +## www[001:006].example.com: +## vars: +## testing1: value1 +## children: +## webservers: +## hosts: +## beta.example.org: diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index c984546621e..5875e9653c7 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -103,7 +103,7 @@ class Inventory(object): # Always create the 'all' and 'ungrouped' groups, even if host_list is # empty: in this case we will subsequently an the implicit 'localhost' to it. - ungrouped = Group(name='ungrouped') + ungrouped = Group('ungrouped') all = Group('all') all.add_child_group(ungrouped) @@ -138,11 +138,12 @@ class Inventory(object): self._vars_plugins = [ x for x in vars_loader.all(self) ] - # get group vars from group_vars/ files and vars plugins - for group in self.groups.values(): + # set group vars from group_vars/ files and vars plugins + for g in self.groups: + group = self.groups[g] group.vars = combine_vars(group.vars, self.get_group_variables(group.name)) - # get host vars from host_vars/ files and vars plugins + # set host vars from host_vars/ files and vars plugins for host in self.get_hosts(): host.vars = combine_vars(host.vars, self.get_host_variables(host.name)) diff --git a/lib/ansible/inventory/dir.py b/lib/ansible/inventory/dir.py index bb077e036b3..29a0c754b85 100644 --- a/lib/ansible/inventory/dir.py +++ b/lib/ansible/inventory/dir.py @@ -24,12 +24,11 @@ import os from ansible import constants as C from ansible.errors import AnsibleError - -from ansible.inventory.host import Host -from ansible.inventory.group import Group from ansible.utils.vars import combine_vars +#FIXME: make into plugins from ansible.inventory.ini import InventoryParser as InventoryINIParser +from ansible.inventory.yaml import InventoryParser as InventoryYAMLParser from ansible.inventory.script import InventoryScript __all__ = ['get_file_parser'] @@ -53,6 +52,8 @@ def get_file_parser(hostsfile, groups, loader): except: pass + #FIXME: make this 'plugin loop' + # script if loader.is_executable(hostsfile): try: parser = InventoryScript(loader=loader, groups=groups, filename=hostsfile) @@ -62,6 +63,19 @@ def get_file_parser(hostsfile, groups, loader): "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile) myerr.append(str(e)) + # YAML/JSON + if not processed and os.path.splitext(hostsfile)[-1] in C.YAML_FILENAME_EXTENSIONS: + try: + parser = InventoryYAMLParser(loader=loader, groups=groups, filename=hostsfile) + processed = True + except Exception as e: + if shebang_present and not loader.is_executable(hostsfile): + myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \ + "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile) + else: + myerr.append(str(e)) + + # ini if not processed: try: parser = InventoryINIParser(loader=loader, groups=groups, filename=hostsfile) diff --git a/lib/ansible/inventory/group.py b/lib/ansible/inventory/group.py index 0c3b5a60b05..482fe2a18fc 100644 --- a/lib/ansible/inventory/group.py +++ b/lib/ansible/inventory/group.py @@ -18,7 +18,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from ansible.errors import AnsibleError -from ansible.utils.debug import debug class Group: ''' a group of ansible hosts ''' @@ -34,6 +33,7 @@ class Group: self.child_groups = [] self.parent_groups = [] self._hosts_cache = None + self.priority = 1 #self.clear_hosts_cache() #if self.name is None: @@ -162,3 +162,10 @@ class Group: return self._get_ancestors().values() + def set_priority(self, priority): + try: + self.priority = int(priority) + except TypeError: + #FIXME: warn about invalid priority + pass + diff --git a/lib/ansible/inventory/ini.py b/lib/ansible/inventory/ini.py index 4d43977004d..defe0517a17 100644 --- a/lib/ansible/inventory/ini.py +++ b/lib/ansible/inventory/ini.py @@ -143,7 +143,10 @@ class InventoryParser(object): # applied to the current group. elif state == 'vars': (k, v) = self._parse_variable_definition(line) - self.groups[groupname].set_variable(k, v) + if k != 'ansible_group_priority': + self.groups[groupname].set_variable(k, v) + else: + self.groups[groupname].set_priority(v) # [groupname:children] contains subgroup names that must be # added as children of the current group. The subgroup names diff --git a/lib/ansible/inventory/yaml.py b/lib/ansible/inventory/yaml.py new file mode 100644 index 00000000000..4317f37122b --- /dev/null +++ b/lib/ansible/inventory/yaml.py @@ -0,0 +1,200 @@ +# Copyright 2016 RedHat, inc +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +############################################# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re + +from ansible import constants as C +from ansible.inventory.host import Host +from ansible.inventory.group import Group +from ansible.inventory.expand_hosts import detect_range +from ansible.inventory.expand_hosts import expand_hostname_range +from ansible.parsing.utils.addresses import parse_address + +class InventoryParser(object): + """ + Takes an INI-format inventory file and builds a list of groups and subgroups + with their associated hosts and variable settings. + """ + + def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST): + self._loader = loader + self.filename = filename + + # Start with an empty host list and whatever groups we're passed in + # (which should include the default 'all' and 'ungrouped' groups). + + self.hosts = {} + self.patterns = {} + self.groups = groups + + # Read in the hosts, groups, and variables defined in the + # inventory file. + data = loader.load_from_file(filename) + + self._parse(data) + + def _parse(self, data): + ''' + Populates self.groups from the given array of lines. Raises an error on + any parse failure. + ''' + + self._compile_patterns() + + # We expect top level keys to correspond to groups, iterate over them + # to get host, vars and subgroups (which we iterate over recursivelly) + for group_name in data.keys(): + self._parse_groups(group_name, data[group_name]) + + # Finally, add all top-level groups as children of 'all'. + # We exclude ungrouped here because it was already added as a child of + # 'all' at the time it was created. + for group in self.groups.values(): + if group.depth == 0 and group.name not in ('all', 'ungrouped'): + self.groups['all'].add_child_group(Group(group_name)) + + def _parse_groups(self, group, group_data): + + if group not in self.groups: + self.groups[group] = Group(name=group) + + if isinstance(group_data, dict): + if 'vars' in group_data: + for var in group_data['vars']: + if var != 'ansible_group_priority': + self.groups[group].set_variable(var, group_data['vars'][var]) + else: + self.groups[group].set_priority(group_data['vars'][var]) + + if 'children' in group_data: + for subgroup in group_data['children']: + self._parse_groups(subgroup, group_data['children'][subgroup]) + self.groups[group].add_child_group(self.groups[subgroup]) + + if 'hosts' in group_data: + for host_pattern in group_data['hosts']: + hosts = self._parse_host(host_pattern, group_data['hosts'][host_pattern]) + for h in hosts: + self.groups[group].add_host(h) + + + def _parse_host(self, host_pattern, host_data): + ''' + Each host key can be a pattern, try to process it and add variables as needed + ''' + (hostnames, port) = self._expand_hostpattern(host_pattern) + hosts = self._Hosts(hostnames, port) + + if isinstance(host_data, dict): + for k in host_data: + for h in hosts: + h.set_variable(k, host_data[k]) + if k in ['ansible_host', 'ansible_ssh_host']: + h.address = host_data[k] + return hosts + + def _expand_hostpattern(self, hostpattern): + ''' + Takes a single host pattern and returns a list of hostnames and an + optional port number that applies to all of them. + ''' + + # Can the given hostpattern be parsed as a host with an optional port + # specification? + + try: + (pattern, port) = parse_address(hostpattern, allow_ranges=True) + except: + # not a recognizable host pattern + pattern = hostpattern + port = None + + # Once we have separated the pattern, we expand it into list of one or + # more hostnames, depending on whether it contains any [x:y] ranges. + + if detect_range(pattern): + hostnames = expand_hostname_range(pattern) + else: + hostnames = [pattern] + + return (hostnames, port) + + def _Hosts(self, hostnames, port): + ''' + Takes a list of hostnames and a port (which may be None) and returns a + list of Hosts (without recreating anything in self.hosts). + ''' + + hosts = [] + + # Note that we decide whether or not to create a Host based solely on + # the (non-)existence of its hostname in self.hosts. This means that one + # cannot add both "foo:22" and "foo:23" to the inventory. + + for hn in hostnames: + if hn not in self.hosts: + self.hosts[hn] = Host(name=hn, port=port) + hosts.append(self.hosts[hn]) + + return hosts + + def get_host_variables(self, host): + return {} + + def _compile_patterns(self): + ''' + Compiles the regular expressions required to parse the inventory and + stores them in self.patterns. + ''' + + # Section names are square-bracketed expressions at the beginning of a + # line, comprising (1) a group name optionally followed by (2) a tag + # that specifies the contents of the section. We ignore any trailing + # whitespace and/or comments. For example: + # + # [groupname] + # [somegroup:vars] + # [naughty:children] # only get coal in their stockings + + self.patterns['section'] = re.compile( + r'''^\[ + ([^:\]\s]+) # group name (see groupname below) + (?::(\w+))? # optional : and tag name + \] + \s* # ignore trailing whitespace + (?:\#.*)? # and/or a comment till the + $ # end of the line + ''', re.X + ) + + # FIXME: What are the real restrictions on group names, or rather, what + # should they be? At the moment, they must be non-empty sequences of non + # whitespace characters excluding ':' and ']', but we should define more + # precise rules in order to support better diagnostics. + + self.patterns['groupname'] = re.compile( + r'''^ + ([^:\]\s]+) + \s* # ignore trailing whitespace + (?:\#.*)? # and/or a comment till the + $ # end of the line + ''', re.X + )