draft add group merge priority and yaml inventory

* now you can specify a yaml invenotry file

* ansible_group_priority will now set this property on groups

* added example yaml inventory

* TODO: make group var merging depend on priority

  groups, child/parent relationships should remain unchanged.

examples/hosts.yaml

@@ -0,0 +1,43 @@
+# This is the default ansible 'hosts' file.
+#
+# It should live in /etc/ansible/hosts
+#
+#   - Comments begin with the '#' character
+#   - Blank lines are ignored
+#   - Top level entries are assumed to be groups
+#   - Hosts must be specified in a group's hosts:
+#     and they must be a key (: terminated)
+#   - groups can have children, hosts and vars keys
+#   - Anything defined under a hosts is assumed to be a var
+#   - You can enter hostnames or ip addresses
+#   - A hostname/ip can be a member of multiple groups
+# Ex 1: Ungrouped hosts, put in 'ungrouped' group
+##ungrouped:
+##  hosts:
+##      green.example.com:
+##          ansible_ssh_host: 191.168.100.32
+##      blue.example.com:
+##      192.168.100.1:
+##      192.168.100.10:
+
+# Ex 2: A collection of hosts belonging to the 'webservers' group
+
+##webservers:
+##  hosts:
+##      alpha.example.org:
+##      beta.example.org:
+##      192.168.1.100:
+##      192.168.1.110:
+
+# Ex 3: You can create hosts using ranges and add children groups and vars to a group
+# The child group can define anything you would normall add to a group
+
+##testing:
+##  hosts:
+##      www[001:006].example.com:
+##  vars:
+##      testing1: value1
+##  children:
+##      webservers:
+##          hosts:
+##              beta.example.org:

lib/ansible/inventory/__init__.py

@@ -103,7 +103,7 @@ class Inventory(object):
         # Always create the 'all' and 'ungrouped' groups, even if host_list is
         # empty: in this case we will subsequently an the implicit 'localhost' to it.
 
-        ungrouped = Group(name='ungrouped')
+        ungrouped = Group('ungrouped')
         all = Group('all')
         all.add_child_group(ungrouped)
 
@@ -138,11 +138,12 @@ class Inventory(object):
 
         self._vars_plugins = [ x for x in vars_loader.all(self) ]
 
-        # get group vars from group_vars/ files and vars plugins
+        # set group vars from group_vars/ files and vars plugins
-        for group in self.groups.values():
+        for g in self.groups:
+            group = self.groups[g]
             group.vars = combine_vars(group.vars, self.get_group_variables(group.name))
 
-        # get host vars from host_vars/ files and vars plugins
+        # set host vars from host_vars/ files and vars plugins
         for host in self.get_hosts():
             host.vars = combine_vars(host.vars, self.get_host_variables(host.name))
 

lib/ansible/inventory/dir.py

@@ -24,12 +24,11 @@ import os
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
-
-from ansible.inventory.host import Host
-from ansible.inventory.group import Group
 from ansible.utils.vars import combine_vars
 
+#FIXME: make into plugins
 from ansible.inventory.ini import InventoryParser as InventoryINIParser
+from ansible.inventory.yaml import InventoryParser as InventoryYAMLParser
 from ansible.inventory.script import InventoryScript
 
 __all__ = ['get_file_parser']
@@ -53,6 +52,8 @@ def get_file_parser(hostsfile, groups, loader):
     except:
         pass
 
+    #FIXME: make this 'plugin loop'
+    # script
     if loader.is_executable(hostsfile):
         try:
             parser = InventoryScript(loader=loader, groups=groups, filename=hostsfile)
@@ -62,6 +63,19 @@ def get_file_parser(hostsfile, groups, loader):
                             "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile)
             myerr.append(str(e))
 
+    # YAML/JSON
+    if not processed and os.path.splitext(hostsfile)[-1] in C.YAML_FILENAME_EXTENSIONS:
+        try:
+            parser = InventoryYAMLParser(loader=loader, groups=groups, filename=hostsfile)
+            processed = True
+        except Exception as e:
+            if shebang_present and not loader.is_executable(hostsfile):
+                myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \
+                              "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile)
+            else:
+                myerr.append(str(e))
+
+    # ini
     if not processed:
         try:
             parser = InventoryINIParser(loader=loader, groups=groups, filename=hostsfile)

lib/ansible/inventory/group.py

@@ -18,7 +18,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible.errors import AnsibleError
-from ansible.utils.debug import debug
 
 class Group:
     ''' a group of ansible hosts '''
@@ -34,6 +33,7 @@ class Group:
         self.child_groups = []
         self.parent_groups = []
         self._hosts_cache = None
+        self.priority = 1
 
         #self.clear_hosts_cache()
         #if self.name is None:
@@ -162,3 +162,10 @@ class Group:
 
         return self._get_ancestors().values()
 
+    def set_priority(self, priority):
+        try:
+            self.priority = int(priority)
+        except TypeError:
+            #FIXME: warn about invalid priority
+            pass
+

lib/ansible/inventory/ini.py

@@ -143,7 +143,10 @@ class InventoryParser(object):
             # applied to the current group.
             elif state == 'vars':
                 (k, v) = self._parse_variable_definition(line)
-                self.groups[groupname].set_variable(k, v)
+                if k != 'ansible_group_priority':
+                    self.groups[groupname].set_variable(k, v)
+                else:
+                    self.groups[groupname].set_priority(v)
 
             # [groupname:children] contains subgroup names that must be
             # added as children of the current group. The subgroup names

lib/ansible/inventory/yaml.py

@@ -0,0 +1,200 @@
+# Copyright 2016 RedHat, inc
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+#############################################
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+
+from ansible import constants as C
+from ansible.inventory.host import Host
+from ansible.inventory.group import Group
+from ansible.inventory.expand_hosts import detect_range
+from ansible.inventory.expand_hosts import expand_hostname_range
+from ansible.parsing.utils.addresses import parse_address
+
+class InventoryParser(object):
+    """
+    Takes an INI-format inventory file and builds a list of groups and subgroups
+    with their associated hosts and variable settings.
+    """
+
+    def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST):
+        self._loader = loader
+        self.filename = filename
+
+        # Start with an empty host list and whatever groups we're passed in
+        # (which should include the default 'all' and 'ungrouped' groups).
+
+        self.hosts = {}
+        self.patterns = {}
+        self.groups = groups
+
+        # Read in the hosts, groups, and variables defined in the
+        # inventory file.
+        data = loader.load_from_file(filename)
+
+        self._parse(data)
+
+    def _parse(self, data):
+        '''
+        Populates self.groups from the given array of lines. Raises an error on
+        any parse failure.
+        '''
+
+        self._compile_patterns()
+
+        # We expect top level keys to correspond to groups, iterate over them
+        # to get host, vars and subgroups (which we iterate over recursivelly)
+        for group_name in data.keys():
+            self._parse_groups(group_name, data[group_name])
+
+        # Finally, add all top-level groups as children of 'all'.
+        # We exclude ungrouped here because it was already added as a child of
+        # 'all' at the time it was created.
+        for group in self.groups.values():
+            if group.depth == 0 and group.name not in ('all', 'ungrouped'):
+                self.groups['all'].add_child_group(Group(group_name))
+
+    def _parse_groups(self, group, group_data):
+
+        if group not in self.groups:
+            self.groups[group] = Group(name=group)
+
+        if isinstance(group_data, dict):
+            if 'vars' in group_data:
+                for var in group_data['vars']:
+                    if var != 'ansible_group_priority':
+                        self.groups[group].set_variable(var, group_data['vars'][var])
+                    else:
+                        self.groups[group].set_priority(group_data['vars'][var])
+
+            if 'children' in group_data:
+                for subgroup in group_data['children']:
+                    self._parse_groups(subgroup, group_data['children'][subgroup])
+                    self.groups[group].add_child_group(self.groups[subgroup])
+
+            if 'hosts' in group_data:
+                for host_pattern in group_data['hosts']:
+                    hosts = self._parse_host(host_pattern, group_data['hosts'][host_pattern])
+                    for h in hosts:
+                        self.groups[group].add_host(h)
+
+
+    def _parse_host(self, host_pattern, host_data):
+        '''
+        Each host key can be a pattern, try to process it and add variables as needed
+        '''
+        (hostnames, port) = self._expand_hostpattern(host_pattern)
+        hosts = self._Hosts(hostnames, port)
+
+        if isinstance(host_data, dict):
+            for k in host_data:
+                for h in hosts:
+                    h.set_variable(k, host_data[k])
+                    if k in ['ansible_host', 'ansible_ssh_host']:
+                        h.address = host_data[k]
+        return hosts
+
+    def _expand_hostpattern(self, hostpattern):
+        '''
+        Takes a single host pattern and returns a list of hostnames and an
+        optional port number that applies to all of them.
+        '''
+
+        # Can the given hostpattern be parsed as a host with an optional port
+        # specification?
+
+        try:
+            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
+        except:
+            # not a recognizable host pattern
+            pattern = hostpattern
+            port = None
+
+        # Once we have separated the pattern, we expand it into list of one or
+        # more hostnames, depending on whether it contains any [x:y] ranges.
+
+        if detect_range(pattern):
+            hostnames = expand_hostname_range(pattern)
+        else:
+            hostnames = [pattern]
+
+        return (hostnames, port)
+
+    def _Hosts(self, hostnames, port):
+        '''
+        Takes a list of hostnames and a port (which may be None) and returns a
+        list of Hosts (without recreating anything in self.hosts).
+        '''
+
+        hosts = []
+
+        # Note that we decide whether or not to create a Host based solely on
+        # the (non-)existence of its hostname in self.hosts. This means that one
+        # cannot add both "foo:22" and "foo:23" to the inventory.
+
+        for hn in hostnames:
+            if hn not in self.hosts:
+                self.hosts[hn] = Host(name=hn, port=port)
+            hosts.append(self.hosts[hn])
+
+        return hosts
+
+    def get_host_variables(self, host):
+        return {}
+
+    def _compile_patterns(self):
+        '''
+        Compiles the regular expressions required to parse the inventory and
+        stores them in self.patterns.
+        '''
+
+        # Section names are square-bracketed expressions at the beginning of a
+        # line, comprising (1) a group name optionally followed by (2) a tag
+        # that specifies the contents of the section. We ignore any trailing
+        # whitespace and/or comments. For example:
+        #
+        # [groupname]
+        # [somegroup:vars]
+        # [naughty:children] # only get coal in their stockings
+
+        self.patterns['section'] = re.compile(
+            r'''^\[
+                    ([^:\]\s]+)             # group name (see groupname below)
+                    (?::(\w+))?             # optional : and tag name
+                \]
+                \s*                         # ignore trailing whitespace
+                (?:\#.*)?                   # and/or a comment till the
+                $                           # end of the line
+            ''', re.X
+        )
+
+        # FIXME: What are the real restrictions on group names, or rather, what
+        # should they be? At the moment, they must be non-empty sequences of non
+        # whitespace characters excluding ':' and ']', but we should define more
+        # precise rules in order to support better diagnostics.
+
+        self.patterns['groupname'] = re.compile(
+            r'''^
+                ([^:\]\s]+)
+                \s*                         # ignore trailing whitespace
+                (?:\#.*)?                   # and/or a comment till the
+                $                           # end of the line
+            ''', re.X
+        )