ansible/test/units/module_utils/facts/test_collector.py
Adrian Likins 6e585bdf24
Fact collector ordering deps (#31362)
Add deps/requires for fact collectors

Fact collectors can now set a required_facts
class attribute that will be a set of the names
of fact collectors they require to be run first.

ie, if a collector needs to know the ansible_distribution,
it should set it's required_facts to include 'distribution'

        required_facts = set(['distribution'])

If a collector requires another collector, it gets added
to the selected collector names.

We then topological sort the ordering of the collectors
so that deps work out (ie, 'distribution' will run before
'service_mgr')

required_facts were added to the collectors for:

        - network (requires 'distribution', 'platform')
        - hardware (requires 'platform')
        - service_mgr (requires 'distribution', 'platform')

Fix name references for facts (need 'ansible_' prefix)
is service_mgr

Fixes #30753
2018-01-22 18:23:40 -05:00

563 lines
26 KiB
Python

# This file is part of Ansible
# -*- coding: utf-8 -*-
#
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
# Make coding more python3-ish
from __future__ import (absolute_import, division)
__metaclass__ = type
from collections import defaultdict
import pprint
# for testing
from ansible.compat.tests import unittest
from ansible.module_utils.facts import collector
from ansible.module_utils.facts import default_collectors
class TestFindCollectorsForPlatform(unittest.TestCase):
def test(self):
compat_platforms = [{'system': 'Generic'}]
res = collector.find_collectors_for_platform(default_collectors.collectors,
compat_platforms)
for coll_class in res:
self.assertIn(coll_class._platform, ('Generic'))
def test_linux(self):
compat_platforms = [{'system': 'Linux'}]
res = collector.find_collectors_for_platform(default_collectors.collectors,
compat_platforms)
for coll_class in res:
self.assertIn(coll_class._platform, ('Linux'))
def test_linux_or_generic(self):
compat_platforms = [{'system': 'Generic'}, {'system': 'Linux'}]
res = collector.find_collectors_for_platform(default_collectors.collectors,
compat_platforms)
for coll_class in res:
self.assertIn(coll_class._platform, ('Generic', 'Linux'))
class TestSelectCollectorNames(unittest.TestCase):
def _assert_equal_detail(self, obj1, obj2, msg=None):
msg = 'objects are not equal\n%s\n\n!=\n\n%s' % (pprint.pformat(obj1), pprint.pformat(obj2))
return self.assertEqual(obj1, obj2, msg)
def test(self):
collector_names = ['distribution', 'all_ipv4_addresses',
'local', 'pkg_mgr']
all_fact_subsets = self._all_fact_subsets()
res = collector.select_collector_classes(collector_names,
all_fact_subsets)
expected = [default_collectors.DistributionFactCollector,
default_collectors.PkgMgrFactCollector]
self._assert_equal_detail(res, expected)
def test_default_collectors(self):
platform_info = {'system': 'Generic'}
compat_platforms = [platform_info]
collectors_for_platform = collector.find_collectors_for_platform(default_collectors.collectors,
compat_platforms)
all_fact_subsets, aliases_map = collector.build_fact_id_to_collector_map(collectors_for_platform)
all_valid_subsets = frozenset(all_fact_subsets.keys())
collector_names = collector.get_collector_names(valid_subsets=all_valid_subsets,
aliases_map=aliases_map,
platform_info=platform_info)
complete_collector_names = collector._solve_deps(collector_names, all_fact_subsets)
dep_map = collector.build_dep_data(complete_collector_names, all_fact_subsets)
ordered_deps = collector.tsort(dep_map)
ordered_collector_names = [x[0] for x in ordered_deps]
res = collector.select_collector_classes(ordered_collector_names,
all_fact_subsets)
self.assertTrue(res.index(default_collectors.ServiceMgrFactCollector) >
res.index(default_collectors.DistributionFactCollector),
res)
self.assertTrue(res.index(default_collectors.ServiceMgrFactCollector) >
res.index(default_collectors.PlatformFactCollector),
res)
def _all_fact_subsets(self, data=None):
all_fact_subsets = defaultdict(list)
_data = {'pkg_mgr': [default_collectors.PkgMgrFactCollector],
'distribution': [default_collectors.DistributionFactCollector],
'network': [default_collectors.LinuxNetworkCollector]}
data = data or _data
for key, value in data.items():
all_fact_subsets[key] = value
return all_fact_subsets
class TestGetCollectorNames(unittest.TestCase):
def test_none(self):
res = collector.get_collector_names()
self.assertIsInstance(res, set)
self.assertEqual(res, set([]))
def test_empty_sets(self):
res = collector.get_collector_names(valid_subsets=frozenset([]),
minimal_gather_subset=frozenset([]),
gather_subset=[])
self.assertIsInstance(res, set)
self.assertEqual(res, set([]))
def test_empty_valid_and_min_with_all_gather_subset(self):
res = collector.get_collector_names(valid_subsets=frozenset([]),
minimal_gather_subset=frozenset([]),
gather_subset=['all'])
self.assertIsInstance(res, set)
self.assertEqual(res, set([]))
def test_one_valid_with_all_gather_subset(self):
valid_subsets = frozenset(['my_fact'])
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=frozenset([]),
gather_subset=['all'])
self.assertIsInstance(res, set)
self.assertEqual(res, set(['my_fact']))
def _compare_res(self, gather_subset1, gather_subset2,
valid_subsets=None, min_subset=None):
valid_subsets = valid_subsets or frozenset()
minimal_gather_subset = min_subset or frozenset()
res1 = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=gather_subset1)
res2 = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=gather_subset2)
return res1, res2
def test_not_all_other_order(self):
valid_subsets = frozenset(['min_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['min_fact'])
res1, res2 = self._compare_res(['!all', 'whatever'],
['whatever', '!all'],
valid_subsets=valid_subsets,
min_subset=minimal_gather_subset)
self.assertEqual(res1, res2)
self.assertEqual(res1, set(['min_fact', 'whatever']))
def test_not_all_other_order_min(self):
valid_subsets = frozenset(['min_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['min_fact'])
res1, res2 = self._compare_res(['!min_fact', 'whatever'],
['whatever', '!min_fact'],
valid_subsets=valid_subsets,
min_subset=minimal_gather_subset)
self.assertEqual(res1, res2)
self.assertEqual(res1, set(['whatever']))
def test_one_minimal_with_all_gather_subset(self):
my_fact = 'my_fact'
valid_subsets = frozenset([my_fact])
minimal_gather_subset = valid_subsets
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['all'])
self.assertIsInstance(res, set)
self.assertEqual(res, set(['my_fact']))
def test_with_all_gather_subset(self):
valid_subsets = frozenset(['my_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['my_fact'])
# even with '!all', the minimal_gather_subset should be returned
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['all'])
self.assertIsInstance(res, set)
self.assertEqual(res, set(['my_fact', 'something_else', 'whatever']))
def test_one_minimal_with_not_all_gather_subset(self):
valid_subsets = frozenset(['my_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['my_fact'])
# even with '!all', the minimal_gather_subset should be returned
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['!all'])
self.assertIsInstance(res, set)
self.assertEqual(res, set(['my_fact']))
def test_gather_subset_excludes(self):
valid_subsets = frozenset(['my_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['min_fact', 'min_another'])
# even with '!all', the minimal_gather_subset should be returned
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
# gather_subset=set(['all', '!my_fact', '!whatever']))
# gather_subset=['all', '!my_fact', '!whatever'])
gather_subset=['!min_fact', '!whatever'])
self.assertIsInstance(res, set)
# min_another is in minimal_gather_subset, so always returned
self.assertEqual(res, set(['min_another']))
def test_gather_subset_excludes_ordering(self):
valid_subsets = frozenset(['my_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['my_fact'])
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['!all', 'whatever'])
self.assertIsInstance(res, set)
# excludes are higher precedence than includes, so !all excludes everything
# and then minimal_gather_subset is added. so '!all', 'other' == '!all'
self.assertEqual(res, set(['my_fact', 'whatever']))
def test_gather_subset_excludes_min(self):
valid_subsets = frozenset(['min_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['min_fact'])
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['whatever', '!min'])
self.assertIsInstance(res, set)
# excludes are higher precedence than includes, so !all excludes everything
# and then minimal_gather_subset is added. so '!all', 'other' == '!all'
self.assertEqual(res, set(['whatever']))
def test_gather_subset_excludes_min_and_all(self):
valid_subsets = frozenset(['min_fact', 'something_else', 'whatever'])
minimal_gather_subset = frozenset(['min_fact'])
res = collector.get_collector_names(valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['whatever', '!all', '!min'])
self.assertIsInstance(res, set)
# excludes are higher precedence than includes, so !all excludes everything
# and then minimal_gather_subset is added. so '!all', 'other' == '!all'
self.assertEqual(res, set(['whatever']))
def test_invaid_gather_subset(self):
valid_subsets = frozenset(['my_fact', 'something_else'])
minimal_gather_subset = frozenset(['my_fact'])
self.assertRaisesRegexp(TypeError,
r'Bad subset .* given to Ansible.*allowed\:.*all,.*my_fact.*',
collector.get_collector_names,
valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=['my_fact', 'not_a_valid_gather_subset'])
class TestFindUnresolvedRequires(unittest.TestCase):
def test(self):
names = ['network', 'virtual', 'env']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.find_unresolved_requires(names, all_fact_subsets)
# pprint.pprint(res)
self.assertIsInstance(res, set)
self.assertEqual(res, set(['platform', 'distribution']))
def test_resolved(self):
names = ['network', 'virtual', 'env', 'platform', 'distribution']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'distribution': [default_collectors.DistributionFactCollector],
'platform': [default_collectors.PlatformFactCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.find_unresolved_requires(names, all_fact_subsets)
# pprint.pprint(res)
self.assertIsInstance(res, set)
self.assertEqual(res, set())
class TestBuildDepData(unittest.TestCase):
def test(self):
names = ['network', 'virtual', 'env']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.build_dep_data(names, all_fact_subsets)
# pprint.pprint(dict(res))
self.assertIsInstance(res, defaultdict)
self.assertEqual(dict(res),
{'network': set(['platform', 'distribution']),
'virtual': set(),
'env': set()})
class TestSolveDeps(unittest.TestCase):
def test_no_solution(self):
unresolved = set(['required_thing1', 'required_thing2'])
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
self.assertRaises(collector.CollectorNotFoundError,
collector._solve_deps,
unresolved,
all_fact_subsets)
def test(self):
unresolved = set(['env', 'network'])
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector],
'platform': [default_collectors.PlatformFactCollector],
'distribution': [default_collectors.DistributionFactCollector]}
res = collector.resolve_requires(unresolved, all_fact_subsets)
res = collector._solve_deps(unresolved, all_fact_subsets)
self.assertIsInstance(res, set)
for goal in unresolved:
self.assertIn(goal, res)
class TestResolveRequires(unittest.TestCase):
def test_no_resolution(self):
unresolved = ['required_thing1', 'required_thing2']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
self.assertRaisesRegexp(collector.UnresolvedFactDep,
'unresolved fact dep.*required_thing2',
collector.resolve_requires,
unresolved, all_fact_subsets)
def test(self):
unresolved = ['env', 'network']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.resolve_requires(unresolved, all_fact_subsets)
for goal in unresolved:
self.assertIn(goal, res)
def test_exception(self):
unresolved = ['required_thing1']
all_fact_subsets = {}
try:
collector.resolve_requires(unresolved, all_fact_subsets)
except collector.UnresolvedFactDep as exc:
self.assertIn(unresolved[0], '%s' % exc)
class TestTsort(unittest.TestCase):
def test(self):
dep_map = {'network': set(['distribution', 'platform']),
'virtual': set(),
'platform': set(['what_platform_wants']),
'what_platform_wants': set(),
'network_stuff': set(['network'])}
res = collector.tsort(dep_map)
# pprint.pprint(res)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertTrue(names.index('network_stuff') > names.index('network'))
self.assertTrue(names.index('platform') > names.index('what_platform_wants'))
self.assertTrue(names.index('network') > names.index('platform'))
def test_cycles(self):
dep_map = {'leaf1': set(),
'leaf2': set(),
'node1': set(['node2']),
'node2': set(['node3']),
'node3': set(['node1'])}
self.assertRaises(collector.CycleFoundInFactDeps,
collector.tsort,
dep_map)
def test_just_nodes(self):
dep_map = {'leaf1': set(),
'leaf4': set(),
'leaf3': set(),
'leaf2': set()}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
# not a lot to assert here, any order of the
# results is valid
self.assertEqual(set(names), set(dep_map.keys()))
def test_self_deps(self):
dep_map = {'node1': set(['node1']),
'node2': set(['node2'])}
self.assertRaises(collector.CycleFoundInFactDeps,
collector.tsort,
dep_map)
def test_unsolvable(self):
dep_map = {'leaf1': set(),
'node2': set(['leaf2'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
def test_chain(self):
dep_map = {'leaf1': set(['leaf2']),
'leaf2': set(['leaf3']),
'leaf3': set(['leaf4']),
'leaf4': set(),
'leaf5': set(['leaf1'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
def test_multi_pass(self):
dep_map = {'leaf1': set(),
'leaf2': set(['leaf3', 'leaf1', 'leaf4', 'leaf5']),
'leaf3': set(['leaf4', 'leaf1']),
'leaf4': set(['leaf1']),
'leaf5': set(['leaf1'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
self.assertTrue(names.index('leaf1') < names.index('leaf2'))
for leaf in ('leaf2', 'leaf3', 'leaf4', 'leaf5'):
self.assertTrue(names.index('leaf1') < names.index(leaf))
class TestCollectorClassesFromGatherSubset(unittest.TestCase):
maxDiff = None
def _classes(self,
all_collector_classes=None,
valid_subsets=None,
minimal_gather_subset=None,
gather_subset=None,
gather_timeout=None,
platform_info=None):
platform_info = platform_info or {'system': 'Linux'}
return collector.collector_classes_from_gather_subset(all_collector_classes=all_collector_classes,
valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=gather_subset,
gather_timeout=gather_timeout,
platform_info=platform_info)
def test_no_args(self):
res = self._classes()
self.assertIsInstance(res, list)
self.assertEqual(res, [])
def test_not_all(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['!all'])
self.assertIsInstance(res, list)
self.assertEqual(res, [])
def test_all(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['all'])
self.assertIsInstance(res, list)
def test_hardware(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['hardware'])
self.assertIsInstance(res, list)
self.assertIn(default_collectors.PlatformFactCollector, res)
self.assertIn(default_collectors.LinuxHardwareCollector, res)
self.assertTrue(res.index(default_collectors.LinuxHardwareCollector) >
res.index(default_collectors.PlatformFactCollector))
def test_network(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['network'])
self.assertIsInstance(res, list)
self.assertIn(default_collectors.DistributionFactCollector, res)
self.assertIn(default_collectors.PlatformFactCollector, res)
self.assertIn(default_collectors.LinuxNetworkCollector, res)
self.assertTrue(res.index(default_collectors.LinuxNetworkCollector) >
res.index(default_collectors.PlatformFactCollector))
self.assertTrue(res.index(default_collectors.LinuxNetworkCollector) >
res.index(default_collectors.DistributionFactCollector))
# self.assertEqual(set(res, [default_collectors.DistributionFactCollector,
# default_collectors.PlatformFactCollector,
# default_collectors.LinuxNetworkCollector])
def test_env(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['env'])
self.assertIsInstance(res, list)
self.assertEqual(res, [default_collectors.EnvFactCollector])
def test_facter(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=set(['env', 'facter']))
self.assertIsInstance(res, list)
self.assertEqual(set(res),
set([default_collectors.EnvFactCollector,
default_collectors.FacterFactCollector]))
def test_facter_ohai(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=set(['env', 'facter', 'ohai']))
self.assertIsInstance(res, list)
self.assertEqual(set(res),
set([default_collectors.EnvFactCollector,
default_collectors.FacterFactCollector,
default_collectors.OhaiFactCollector]))
def test_just_facter(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=set(['facter']))
self.assertIsInstance(res, list)
self.assertEqual(set(res),
set([default_collectors.FacterFactCollector]))
def test_collector_specified_multiple_times(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['platform', 'all', 'machine'])
self.assertIsInstance(res, list)
self.assertIn(default_collectors.PlatformFactCollector,
res)
def test_unknown_collector(self):
# something claims 'unknown_collector' is a valid gather_subset, but there is
# no FactCollector mapped to 'unknown_collector'
self.assertRaisesRegexp(TypeError,
r'Bad subset.*unknown_collector.*given to Ansible.*allowed\:.*all,.*env.*',
self._classes,
all_collector_classes=default_collectors.collectors,
gather_subset=['env', 'unknown_collector'])