From ea2f37d2536300b2cf00366e730ca04e9252d45d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 12 Feb 2021 11:14:50 -0500 Subject: [PATCH] allow constructed to use vars plugin (#73418) Allow constructed to optionally use vars plugin data * mostly for those looking to leverage group_vars/ and host_vars/ * limited to already processed sources --- .../fragments/constructed_vars_plugins.yml | 2 + lib/ansible/inventory/data.py | 3 ++ lib/ansible/inventory/manager.py | 4 +- lib/ansible/plugins/inventory/__init__.py | 10 ++-- lib/ansible/plugins/inventory/constructed.py | 52 ++++++++++++++++--- .../invs/1/group_vars/stuff.yml | 1 + .../invs/1/host_vars/testing.yml | 1 + .../inventory_constructed/invs/1/one.yml | 5 ++ .../invs/2/constructed.yml | 7 +++ .../targets/inventory_constructed/runme.sh | 7 +++ 10 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 changelogs/fragments/constructed_vars_plugins.yml create mode 100644 test/integration/targets/inventory_constructed/invs/1/group_vars/stuff.yml create mode 100644 test/integration/targets/inventory_constructed/invs/1/host_vars/testing.yml create mode 100644 test/integration/targets/inventory_constructed/invs/1/one.yml create mode 100644 test/integration/targets/inventory_constructed/invs/2/constructed.yml diff --git a/changelogs/fragments/constructed_vars_plugins.yml b/changelogs/fragments/constructed_vars_plugins.yml new file mode 100644 index 00000000000..9383890b7e7 --- /dev/null +++ b/changelogs/fragments/constructed_vars_plugins.yml @@ -0,0 +1,2 @@ +minor_changes: + - The constructed inventory plugin has new option to force using vars plugins on previouslly processed inventory sources. diff --git a/lib/ansible/inventory/data.py b/lib/ansible/inventory/data.py index df4af766cd6..cf6fbb66acd 100644 --- a/lib/ansible/inventory/data.py +++ b/lib/ansible/inventory/data.py @@ -51,6 +51,7 @@ class InventoryData(object): self.localhost = None self.current_source = None + self.processed_sources = [] # Always create the 'all' and 'ungrouped' groups, for group in ('all', 'ungrouped'): @@ -64,6 +65,7 @@ class InventoryData(object): 'hosts': self.hosts, 'local': self.localhost, 'source': self.current_source, + 'processed_sources': self.processed_sources } return data @@ -73,6 +75,7 @@ class InventoryData(object): self.groups = data.get('groups') self.localhost = data.get('local') self.current_source = data.get('source') + self.processed_sources = data.get('processed_sources') def _create_implicit_localhost(self, pattern): diff --git a/lib/ansible/inventory/manager.py b/lib/ansible/inventory/manager.py index 8849a8c185a..3e70ee48b79 100644 --- a/lib/ansible/inventory/manager.py +++ b/lib/ansible/inventory/manager.py @@ -307,7 +307,9 @@ class InventoryManager(object): else: display.vvv("%s declined parsing %s as it did not pass its verify_file() method" % (plugin_name, source)) - if not parsed: + if parsed: + self._inventory.processed_sources.append(self._inventory.current_source) + else: # only warn/error if NOT using the default or using it and the file is present # TODO: handle 'non file' inventorya and detect vs hardcode default if source != '/etc/ansible/hosts' or os.path.exists(source): diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 3748a80fa50..c718e930f34 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -381,11 +381,12 @@ class Constructable(object): continue self.inventory.set_variable(host, varname, composite) - def _add_host_to_composed_groups(self, groups, variables, host, strict=False): + def _add_host_to_composed_groups(self, groups, variables, host, strict=False, fetch_hostvars=True): ''' helper to create complex groups for plugins based on jinja2 conditionals, hosts that meet the conditional are added to group''' # process each 'group entry' if groups and isinstance(groups, dict): - variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) + if fetch_hostvars: + variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) self.templar.available_variables = variables for group_name in groups: conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % groups[group_name] @@ -403,13 +404,14 @@ class Constructable(object): # add host to group self.inventory.add_child(group_name, host) - def _add_host_to_keyed_groups(self, keys, variables, host, strict=False): + def _add_host_to_keyed_groups(self, keys, variables, host, strict=False, fetch_hostvars=True): ''' helper to create groups for plugins based on variable values and add the corresponding hosts to it''' if keys and isinstance(keys, list): for keyed in keys: if keyed and isinstance(keyed, dict): - variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) + if fetch_hostvars: + variables = combine_vars(variables, self.inventory.get_host(host).get_vars()) try: key = self._compose(keyed.get('key'), variables) except Exception as e: diff --git a/lib/ansible/plugins/inventory/constructed.py b/lib/ansible/plugins/inventory/constructed.py index 424eb00e977..ee8d0df3ad1 100644 --- a/lib/ansible/plugins/inventory/constructed.py +++ b/lib/ansible/plugins/inventory/constructed.py @@ -19,6 +19,16 @@ DOCUMENTATION = ''' description: token that ensures this is a source file for the 'constructed' plugin. required: True choices: ['constructed'] + use_vars_plugins: + description: + - Normally, for performance reasons, vars plugins get executed after the inventory sources complete the base inventory, + this option allows for getting vars related to hosts/groups from those plugins. + - The host_group_vars (enabled by default) 'vars plugin' is the one responsible for reading host_vars/ and group_vars/ directories. + - This will not execute plugins that are not supposed to execute at the 'inventory' stage, see vars plugins docs for details. + required: false + default: false + type: boolean + version_added: '2.11' extends_documentation_fragment: - constructed ''' @@ -70,12 +80,13 @@ EXAMPLES = r''' import os from ansible import constants as C -from ansible.errors import AnsibleParserError +from ansible.errors import AnsibleParserError, AnsibleOptionsError from ansible.inventory.helpers import get_group_vars from ansible.plugins.inventory import BaseInventoryPlugin, Constructable from ansible.module_utils._text import to_native from ansible.utils.vars import combine_vars from ansible.vars.fact_cache import FactCache +from ansible.vars.plugins import get_vars_from_inventory_sources class InventoryModule(BaseInventoryPlugin, Constructable): @@ -100,6 +111,28 @@ class InventoryModule(BaseInventoryPlugin, Constructable): return valid + def get_all_host_vars(self, host, loader, sources): + ''' requires host object ''' + return combine_vars(self.host_groupvars(host, loader, sources), self.host_vars(host, loader, sources)) + + def host_groupvars(self, host, loader, sources): + ''' requires host object ''' + gvars = get_group_vars(host.get_groups()) + + if self.get_option('use_vars_plugins'): + gvars = combine_vars(gvars, get_vars_from_inventory_sources(loader, sources, host.get_groups(), 'all')) + + return gvars + + def host_vars(self, host, loader, sources): + ''' requires host object ''' + hvars = host.get_vars() + + if self.get_option('use_vars_plugins'): + hvars = combine_vars(hvars, get_vars_from_inventory_sources(loader, sources, [host], 'all')) + + return hvars + def parse(self, inventory, loader, path, cache=False): ''' parses the inventory file ''' @@ -107,6 +140,13 @@ class InventoryModule(BaseInventoryPlugin, Constructable): self._read_config_data(path) + sources = [] + try: + sources = inventory.processed_sources + except AttributeError: + if self.get_option('use_vars_plugins'): + raise AnsibleOptionsError("The option use_vars_plugins requires ansible >= 2.11.") + strict = self.get_option('strict') fact_cache = FactCache() try: @@ -114,7 +154,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable): for host in inventory.hosts: # get available variables to templar - hostvars = combine_vars(get_group_vars(inventory.hosts[host].get_groups()), inventory.hosts[host].get_vars()) + hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources) if host in fact_cache: # adds facts if cache is active hostvars = combine_vars(hostvars, fact_cache[host]) @@ -122,15 +162,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable): self._set_composite_vars(self.get_option('compose'), hostvars, host, strict=strict) # refetch host vars in case new ones have been created above - hostvars = combine_vars(get_group_vars(inventory.hosts[host].get_groups()), inventory.hosts[host].get_vars()) + hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources) if host in self._cache: # adds facts if cache is active hostvars = combine_vars(hostvars, self._cache[host]) # constructed groups based on conditionals - self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host, strict=strict) + self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host, strict=strict, fetch_hostvars=False) # constructed groups based variable values - self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host, strict=strict) + self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host, strict=strict, fetch_hostvars=False) except Exception as e: - raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e))) + raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e)), orig_exc=e) diff --git a/test/integration/targets/inventory_constructed/invs/1/group_vars/stuff.yml b/test/integration/targets/inventory_constructed/invs/1/group_vars/stuff.yml new file mode 100644 index 00000000000..a67b90f4c3f --- /dev/null +++ b/test/integration/targets/inventory_constructed/invs/1/group_vars/stuff.yml @@ -0,0 +1 @@ +iamdefined: group4testing diff --git a/test/integration/targets/inventory_constructed/invs/1/host_vars/testing.yml b/test/integration/targets/inventory_constructed/invs/1/host_vars/testing.yml new file mode 100644 index 00000000000..0ffe382138a --- /dev/null +++ b/test/integration/targets/inventory_constructed/invs/1/host_vars/testing.yml @@ -0,0 +1 @@ +hola: lola diff --git a/test/integration/targets/inventory_constructed/invs/1/one.yml b/test/integration/targets/inventory_constructed/invs/1/one.yml new file mode 100644 index 00000000000..ad5a5e0a5fd --- /dev/null +++ b/test/integration/targets/inventory_constructed/invs/1/one.yml @@ -0,0 +1,5 @@ +all: + children: + stuff: + hosts: + testing: diff --git a/test/integration/targets/inventory_constructed/invs/2/constructed.yml b/test/integration/targets/inventory_constructed/invs/2/constructed.yml new file mode 100644 index 00000000000..7c62ef1de54 --- /dev/null +++ b/test/integration/targets/inventory_constructed/invs/2/constructed.yml @@ -0,0 +1,7 @@ +plugin: constructed +use_vars_plugins: true +keyed_groups: + - key: iamdefined + prefix: c + - key: hola + prefix: c diff --git a/test/integration/targets/inventory_constructed/runme.sh b/test/integration/targets/inventory_constructed/runme.sh index 7a36be017e7..0cd1a29311a 100755 --- a/test/integration/targets/inventory_constructed/runme.sh +++ b/test/integration/targets/inventory_constructed/runme.sh @@ -23,3 +23,10 @@ grep '@key0separatorvalue0' out.txt grep '@prefix_hostvalue1' out.txt grep '@prefix_item0' out.txt grep '@prefix_key0_value0' out.txt + + +# test using use_vars_plugins +ansible-inventory -i invs/1/one.yml -i invs/2/constructed.yml --graph | tee out.txt + +grep '@c_lola' out.txt +grep '@c_group4testing' out.txt