From 8940732b580822b4df5c18d1f21907bbc4f1e0cb Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 8 Mar 2019 13:08:37 -0500 Subject: [PATCH] Configurable and parallel gather facts (#49399) * Configurable list of facts modules (#31783) - allow for args dict for specific modules - add way to pass parameters - avoid facts poluting test - move to 'facts gathered' flag - add 'gathering' setting tests - allow parallel option in case serialization is too slow - added support to automatically map network facts uses "smart" connection mapping --- lib/ansible/config/base.yml | 26 ++++ lib/ansible/executor/play_iterator.py | 4 +- lib/ansible/modules/system/gather_facts.py | 48 ++++++++ lib/ansible/plugins/action/gather_facts.py | 112 ++++++++++++++++++ lib/ansible/plugins/loader.py | 2 +- test/integration/targets/gathering/aliases | 1 + .../targets/gathering/explicit.yml | 14 +++ .../targets/gathering/implicit.yml | 23 ++++ test/integration/targets/gathering/runme.sh | 7 ++ test/integration/targets/gathering/smart.yml | 23 ++++ test/integration/targets/gathering/uuid.fact | 10 ++ .../pull/pull-integration-test/local.yml | 1 + 12 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 lib/ansible/modules/system/gather_facts.py create mode 100644 lib/ansible/plugins/action/gather_facts.py create mode 100644 test/integration/targets/gathering/aliases create mode 100644 test/integration/targets/gathering/explicit.yml create mode 100644 test/integration/targets/gathering/implicit.yml create mode 100755 test/integration/targets/gathering/runme.sh create mode 100644 test/integration/targets/gathering/smart.yml create mode 100644 test/integration/targets/gathering/uuid.fact diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 953cae8661d..a4d302a50cb 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1324,6 +1324,32 @@ ERROR_ON_MISSING_HANDLER: ini: - {key: error_on_missing_handler, section: defaults} type: boolean +CONNECTION_FACTS_MODULES: + name: Map of connections to fact modules + default: + eos: eos_facts + frr: frr_facts + ios: ios_facts + iosxr: iosxr_facts + junos: junos_facts + nxos: nxos_facts + vyos: vyos_facts + description: "Which modules to run during a play's fact gathering stage based on connection" + env: [{name: ANSIBLE_CONNECTION_FACTS_MODULES}] + ini: + - {key: connection_facts_modules, section: defaults} + type: dict +FACTS_MODULES: + name: Gather Facts Modules + default: + - smart + description: "Which modules to run during a play's fact gathering stage, using the default of 'smart' will try to figure it out based on connection type." + env: [{name: ANSIBLE_FACTS_MODULES}] + ini: + - {key: facts_modules, section: defaults} + type: list + vars: + - name: ansible_facts_modules GALAXY_IGNORE_CERTS: name: Galaxy validate certs default: False diff --git a/lib/ansible/executor/play_iterator.py b/lib/ansible/executor/play_iterator.py index 5a32a1d52ba..51615ff618e 100644 --- a/lib/ansible/executor/play_iterator.py +++ b/lib/ansible/executor/play_iterator.py @@ -160,7 +160,7 @@ class PlayIterator: # the others. setup_block.run_once = False setup_task = Task(block=setup_block) - setup_task.action = 'setup' + setup_task.action = 'gather_facts' setup_task.name = 'Gathering Facts' setup_task.args = { 'gather_subset': gather_subset, @@ -287,7 +287,7 @@ class PlayIterator: if (gathering == 'implicit' and implied) or \ (gathering == 'explicit' and boolean(self._play.gather_facts, strict=False)) or \ - (gathering == 'smart' and implied and not (self._variable_manager._fact_cache.get(host.name, {}).get('module_setup', False))): + (gathering == 'smart' and implied and not (self._variable_manager._fact_cache.get(host.name, {}).get('_ansible_facts_gathered', False))): # The setup block is always self._blocks[0], as we inject it # during the play compilation in __init__ above. setup_block = self._blocks[0] diff --git a/lib/ansible/modules/system/gather_facts.py b/lib/ansible/modules/system/gather_facts.py new file mode 100644 index 00000000000..711c840a989 --- /dev/null +++ b/lib/ansible/modules/system/gather_facts.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'core'} + + +DOCUMENTATION = ''' +--- +module: gather_facts +version_added: 2.8 +short_description: Gathers facts about remote hosts +description: + - This module takes care of executing the configured facts modules, the default is to use the M(setup) module. + - This module is automatically called by playbooks to gather useful variables about remote hosts that can be used in playbooks. + - It can also be executed directly by C(/usr/bin/ansible) to check what variables are available to a host. + - Ansible provides many I(facts) about the system, automatically. +options: + parallel: + description: + - A toggle that controls if the fact modules are executed in parallel or serially and in order. + This can guarantee the merge order of module facts at the expense of performance. + - By default it will be true if more than one fact module is used. + type: bool +notes: + - This module is mostly a wrapper around other fact gathering modules. + - Options passed to this module must be supported by all the underlying fact modules configured. + - Facts returned by each module will be merged, conflicts will favor 'last merged'. + Order is not guaranteed, when doing parallel gathering on multiple modules. +author: + - "Ansible Core Team" +''' + +RETURN = """ +# depends on the fact module called +""" + +EXAMPLES = """ +# Display facts from all hosts and store them indexed by I(hostname) at C(/tmp/facts). +# ansible all -m gather_facts --tree /tmp/facts +""" diff --git a/lib/ansible/plugins/action/gather_facts.py b/lib/ansible/plugins/action/gather_facts.py new file mode 100644 index 00000000000..9279d2a046e --- /dev/null +++ b/lib/ansible/plugins/action/gather_facts.py @@ -0,0 +1,112 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import time + +from ansible import constants as C +from ansible.plugins.action import ActionBase +from ansible.utils.vars import combine_vars + + +class ActionModule(ActionBase): + + def _get_module_args(self, fact_module, task_vars): + + mod_args = self._task.args.copy() + + # deal with 'setup specific arguments' + if fact_module != 'setup': + + # network facts modules must support gather_subset + if self._connection._load_name not in ('network_cli', 'httpapi', 'netconf'): + subset = mod_args.pop('gather_subset', None) + if subset not in ('all', ['all']): + self._display.warning('Ignoring subset(%s) for %s' % (subset, fact_module)) + + timeout = mod_args.pop('gather_timeout', None) + if timeout is not None: + self._display.warning('Ignoring timeout(%s) for %s' % (timeout, fact_module)) + + fact_filter = mod_args.pop('filter', None) + if fact_filter is not None: + self._display.warning('Ignoring filter(%s) for %s' % (fact_filter, fact_module)) + + return mod_args + + def run(self, tmp=None, task_vars=None): + + self._supports_check_mode = True + + result = super(ActionModule, self).run(tmp, task_vars) + result['ansible_facts'] = {} + + modules = C.config.get_config_value('FACTS_MODULES', variables=task_vars) + parallel = task_vars.pop('ansible_facts_parallel', self._task.args.pop('parallel', None)) + + if 'smart' in modules: + connection_map = C.config.get_config_value('CONNECTION_FACTS_MODULES', variables=task_vars) + modules.extend([connection_map.get(self._connection._load_name, 'setup')]) + modules.pop(modules.index('smart')) + + failed = {} + skipped = {} + if parallel is False or (len(modules) == 1 and parallel is None): + # serially execute each module + for fact_module in modules: + # just one module, no need for fancy async + mod_args = self._get_module_args(fact_module, task_vars) + res = self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=False) + if res.get('failed', False): + failed[fact_module] = res.get('msg') + elif res.get('skipped', False): + skipped[fact_module] = res.get('msg') + else: + result = combine_vars(result, {'ansible_facts': res.get('ansible_facts', {})}) + else: + # do it async + jobs = {} + for fact_module in modules: + + mod_args = self._get_module_args(fact_module, task_vars) + self._display.vvvv("Running %s" % fact_module) + jobs[fact_module] = (self._execute_module(module_name=fact_module, module_args=mod_args, task_vars=task_vars, wrap_async=True)) + + while jobs: + for module in jobs: + poll_args = {'jid': jobs[module]['ansible_job_id'], '_async_dir': os.path.dirname(jobs[module]['results_file'])} + res = self._execute_module(module_name='async_status', module_args=poll_args, task_vars=task_vars, wrap_async=False) + if res.get('finished', 0) == 1: + if res.get('failed', False): + failed[module] = res.get('msg') + elif res.get('skipped', False): + skipped[module] = res.get('msg') + else: + result = combine_vars(result, {'ansible_facts': res.get('ansible_facts', {})}) + del jobs[module] + break + else: + time.sleep(0.1) + else: + time.sleep(0.5) + + if skipped: + result['msg'] = "The following modules were skipped: %s\n" % (', '.join(skipped.keys())) + for skip in skipped: + result['msg'] += ' %s: %s\n' % (skip, skipped[skip]) + if len(skipped) == len(modules): + result['skipped'] = True + + if failed: + result['failed'] = True + result['msg'] = "The following modules failed to execute: %s\n" % (', '.join(failed.keys())) + for fail in failed: + result['msg'] += ' %s: %s\n' % (fail, failed[fail]) + + # tell executor facts were gathered + result['ansible_facts']['_ansible_facts_gathered'] = True + + return result diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 29d41dbb865..8defa773bf2 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -376,7 +376,7 @@ class PluginLoader: from ansible.vars.reserved import is_reserved_name plugin = self._find_plugin(name, mod_type=mod_type, ignore_deprecated=ignore_deprecated, check_aliases=check_aliases) - if plugin and self.package == 'ansible.modules' and is_reserved_name(name): + if plugin and self.package == 'ansible.modules' and name not in ('gather_facts',) and is_reserved_name(name): raise AnsibleError( 'Module "%s" shadows the name of a reserved keyword. Please rename or remove this module. Found at %s' % (name, plugin) ) diff --git a/test/integration/targets/gathering/aliases b/test/integration/targets/gathering/aliases new file mode 100644 index 00000000000..b59832142f2 --- /dev/null +++ b/test/integration/targets/gathering/aliases @@ -0,0 +1 @@ +shippable/posix/group3 diff --git a/test/integration/targets/gathering/explicit.yml b/test/integration/targets/gathering/explicit.yml new file mode 100644 index 00000000000..453dfb6aa0f --- /dev/null +++ b/test/integration/targets/gathering/explicit.yml @@ -0,0 +1,14 @@ +- hosts: testhost + tasks: + - name: ensure facts have not been collected + assert: + that: + - ansible_facts is undefined or not 'fqdn' in ansible_facts + +- hosts: testhost + gather_facts: True + tasks: + - name: ensure facts have been collected + assert: + that: + - ansible_facts is defined and 'fqdn' in ansible_facts diff --git a/test/integration/targets/gathering/implicit.yml b/test/integration/targets/gathering/implicit.yml new file mode 100644 index 00000000000..f1ea965d32d --- /dev/null +++ b/test/integration/targets/gathering/implicit.yml @@ -0,0 +1,23 @@ +- hosts: testhost + tasks: + - name: check that facts were gathered but no local facts exist + assert: + that: + - ansible_facts is defined and 'fqdn' in ansible_facts + - not 'uuid' in ansible_local + - name: create 'local facts' for next gathering + copy: + src: uuid.fact + dest: /etc/ansible/facts.d/ + mode: 0755 + +- hosts: testhost + tasks: + - name: ensure facts are gathered and includes the new 'local facts' created above + assert: + that: + - ansible_facts is defined and 'fqdn' in ansible_facts + - "'uuid' in ansible_local" + + - name: cleanup 'local facts' from target + file: path=/etc/ansible/facts.d/uuid.fact state=absent diff --git a/test/integration/targets/gathering/runme.sh b/test/integration/targets/gathering/runme.sh new file mode 100755 index 00000000000..1c0832c5a93 --- /dev/null +++ b/test/integration/targets/gathering/runme.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eux + +ANSIBLE_GATHERING=smart ansible-playbook smart.yml --flush-cache -i ../../inventory -v "$@" +ANSIBLE_GATHERING=implicit ansible-playbook implicit.yml --flush-cache -i ../../inventory -v "$@" +ANSIBLE_GATHERING=explicit ansible-playbook explicit.yml --flush-cache -i ../../inventory -v "$@" diff --git a/test/integration/targets/gathering/smart.yml b/test/integration/targets/gathering/smart.yml new file mode 100644 index 00000000000..735cb461be6 --- /dev/null +++ b/test/integration/targets/gathering/smart.yml @@ -0,0 +1,23 @@ +- hosts: testhost + tasks: + - name: ensure facts are gathered but no local exists + assert: + that: + - ansible_facts is defined and 'fqdn' in ansible_facts + - not 'uuid' in ansible_local + - name: create local facts for latter test + copy: + src: uuid.fact + dest: /etc/ansible/facts.d/ + mode: 0755 + +- hosts: testhost + tasks: + - name: ensure we still have facts, but didnt pickup new local ones + assert: + that: + - ansible_facts is defined and 'fqdn' in ansible_facts + - not 'uuid' in ansible_local + + - name: remove local facts file + file: path=/etc/ansible/facts.d/uuid.fact state=absent diff --git a/test/integration/targets/gathering/uuid.fact b/test/integration/targets/gathering/uuid.fact new file mode 100644 index 00000000000..79e3f62677e --- /dev/null +++ b/test/integration/targets/gathering/uuid.fact @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import json +import uuid + + +# return a random string +print(json.dumps(str(uuid.uuid4()))) diff --git a/test/integration/targets/pull/pull-integration-test/local.yml b/test/integration/targets/pull/pull-integration-test/local.yml index 3a924fa3fe4..d358ee86863 100644 --- a/test/integration/targets/pull/pull-integration-test/local.yml +++ b/test/integration/targets/pull/pull-integration-test/local.yml @@ -1,5 +1,6 @@ - name: test playbook for ansible-pull hosts: all + gather_facts: False tasks: - name: debug output debug: msg="test task"