diff --git a/test/runner/lib/classification.py b/test/runner/lib/classification.py index 63ca804eab2..b576cfacb83 100644 --- a/test/runner/lib/classification.py +++ b/test/runner/lib/classification.py @@ -130,19 +130,22 @@ class PathMapper(object): :type path: str :rtype: list[str] """ - name, ext = os.path.splitext(os.path.split(path)[1]) + ext = os.path.splitext(os.path.split(path)[1])[1] if path.startswith('lib/ansible/module_utils/'): if ext == '.py': - return self.get_python_module_utils_usage(name) + return self.get_python_module_utils_usage(path) return [] - def get_python_module_utils_usage(self, name): + def get_python_module_utils_usage(self, path): """ - :type name: str + :type path: str :rtype: list[str] """ + if path == 'lib/ansible/module_utils/__init__.py': + return [] + if not self.python_module_utils_imports: display.info('Analyzing python module_utils imports...') before = time.time() @@ -150,7 +153,12 @@ class PathMapper(object): after = time.time() display.info('Processed %d python module_utils in %d second(s).' % (len(self.python_module_utils_imports), after - before)) - return sorted(self.python_module_utils_imports.get(name, set())) + name = os.path.splitext(path)[0].replace('/', '.')[4:] + + if name.endswith('.__init__'): + name = name[:-9] + + return sorted(self.python_module_utils_imports[name]) def classify(self, path): """ diff --git a/test/runner/lib/import_analysis.py b/test/runner/lib/import_analysis.py index aeebccf6e8d..c77d9e499ef 100644 --- a/test/runner/lib/import_analysis.py +++ b/test/runner/lib/import_analysis.py @@ -11,14 +11,20 @@ from lib.util import ( ApplicationError, ) +VIRTUAL_PACKAGES = set([ + 'ansible.module_utils.six', +]) + def get_python_module_utils_imports(compile_targets): - """Return a dictionary of python file paths mapped to sets of module_utils names. + """Return a dictionary of module_utils names mapped to sets of python file paths. :type compile_targets: list[TestTarget] :rtype: dict[str, set[str]] """ - module_utils_files = (os.path.splitext(filename) for filename in os.listdir('lib/ansible/module_utils')) - module_utils = sorted(name[0] for name in module_utils_files if name[0] != '__init__' and name[1] == '.py') + + module_utils = enumerate_module_utils() + virtual_utils = set(m for m in module_utils if any(m.startswith('%s.' % v) for v in VIRTUAL_PACKAGES)) + module_utils -= virtual_utils imports_by_target_path = {} @@ -39,9 +45,33 @@ def get_python_module_utils_imports(compile_targets): results = set([import_name]) - import_path = os.path.join('lib/ansible/module_utils', '%s.py' % import_name) + # virtual packages depend on the modules they contain instead of the reverse + if import_name in VIRTUAL_PACKAGES: + for sub_import in sorted(virtual_utils): + if sub_import.startswith('%s.' % import_name): + if sub_import in seen: + continue + + seen.add(sub_import) + + matches = sorted(recurse_import(sub_import, depth + 1, seen)) + + for result in matches: + results.add(result) + + import_path = os.path.join('lib/', '%s.py' % import_name.replace('.', '/')) + + if import_path not in imports_by_target_path: + import_path = os.path.join('lib/', import_name.replace('.', '/'), '__init__.py') + + if import_path not in imports_by_target_path: + raise ApplicationError('Cannot determine path for module_utils import: %s' % import_name) + + # process imports in reverse so the deepest imports come first + for name in sorted(imports_by_target_path[import_path], reverse=True): + if name in virtual_utils: + continue - for name in sorted(imports_by_target_path.get(import_path, set())): if name in seen: continue @@ -67,12 +97,18 @@ def get_python_module_utils_imports(compile_targets): display.info('%s inherits import %s via %s' % (target_path, module_util_import, module_util), verbosity=6) imports_by_target_path[target_path].add(module_util_import) - imports = dict([(module_util, set()) for module_util in module_utils]) + imports = dict([(module_util, set()) for module_util in module_utils | virtual_utils]) for target_path in imports_by_target_path: for module_util in imports_by_target_path[target_path]: imports[module_util].add(target_path) + # for purposes of mapping module_utils to paths, treat imports of virtual utils the same as the parent package + for virtual_util in virtual_utils: + parent_package = '.'.join(virtual_util.split('.')[:-1]) + imports[virtual_util] = imports[parent_package] + display.info('%s reports imports from parent package %s' % (virtual_util, parent_package), verbosity=6) + for module_util in sorted(imports): if not len(imports[module_util]): display.warning('No imports found which use the "%s" module_util.' % module_util) @@ -80,6 +116,34 @@ def get_python_module_utils_imports(compile_targets): return imports +def enumerate_module_utils(): + """Return a list of available module_utils imports. + :rtype: set[str] + """ + module_utils = [] + base_path = 'lib/ansible/module_utils' + + for root, _, file_names in os.walk(base_path): + for file_name in file_names: + path = os.path.join(root, file_name) + name, ext = os.path.splitext(file_name) + + if path == 'lib/ansible/module_utils/__init__.py': + continue + + if ext != '.py': + continue + + if name == '__init__': + module_util = root + else: + module_util = os.path.join(root, name) + + module_utils.append(module_util[4:].replace('/', '.')) + + return set(module_utils) + + def extract_python_module_utils_imports(path, module_utils): """Return a list of module_utils imports found in the specified source file. :type path: str @@ -110,12 +174,21 @@ class ModuleUtilFinder(ast.NodeVisitor): :type path: str :type module_utils: set[str] """ - super(ModuleUtilFinder, self).__init__() - self.path = path self.module_utils = module_utils self.imports = set() + # implicitly import parent package + + if path.endswith('/__init__.py'): + path = os.path.split(path)[0] + + if path.startswith('lib/ansible/module_utils/'): + package = os.path.split(path)[0].replace('/', '.')[4:] + + if package != 'ansible.module_utils' and package not in VIRTUAL_PACKAGES: + self.add_import(package, 0) + # noinspection PyPep8Naming # pylint: disable=locally-disabled, invalid-name def visit_Import(self, node): @@ -127,7 +200,7 @@ class ModuleUtilFinder(ast.NodeVisitor): for alias in node.names: if alias.name.startswith('ansible.module_utils.'): # import ansible.module_utils.MODULE[.MODULE] - self.add_import(alias.name.split('.')[2], node.lineno) + self.add_import(alias.name, node.lineno) # noinspection PyPep8Naming # pylint: disable=locally-disabled, invalid-name @@ -140,28 +213,30 @@ class ModuleUtilFinder(ast.NodeVisitor): if not node.module: return - if node.module == 'ansible.module_utils': + if node.module == 'ansible.module_utils' or node.module.startswith('ansible.module_utils.'): for alias in node.names: # from ansible.module_utils import MODULE[, MODULE] - self.add_import(alias.name, node.lineno) - elif node.module.startswith('ansible.module_utils.'): - # from ansible.module_utils.MODULE[.MODULE] - self.add_import(node.module.split('.')[2], node.lineno) + # from ansible.module_utils.MODULE[.MODULE] import MODULE[, MODULE] + self.add_import('%s.%s' % (node.module, alias.name), node.lineno) def add_import(self, name, line_number): """ :type name: str :type line_number: int """ - if name in self.imports: - return # duplicate imports are ignored + import_name = name - if name not in self.module_utils: - if self.path.startswith('test/'): - return # invalid imports in tests are ignored + while len(name) > len('ansible.module_utils.'): + if name in self.module_utils: + if name not in self.imports: + display.info('%s:%d imports module_utils: %s' % (self.path, line_number, name), verbosity=5) + self.imports.add(name) - raise Exception('%s:%d Invalid module_util import: %s' % (self.path, line_number, name)) + return # duplicate imports are ignored - display.info('%s:%d imports module_utils: %s' % (self.path, line_number, name), verbosity=5) + name = '.'.join(name.split('.')[:-1]) - self.imports.add(name) + if self.path.startswith('test/'): + return # invalid imports in tests are ignored + + raise ApplicationError('%s:%d Invalid module_utils import: %s' % (self.path, line_number, import_name))