From 0f5a63f1b99e586f86932907c49b1e8877128957 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 27 Mar 2020 22:19:49 -0400 Subject: [PATCH] ansilbe-doc list collections plugins (#67928) Now -l and -F will list plugins from discover-able collections --- lib/ansible/cli/doc.py | 23 ++++++++- lib/ansible/constants.py | 1 + lib/ansible/utils/collection_loader.py | 65 +++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 12bb590fb4c..30adf40880a 100644 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -26,7 +26,7 @@ from ansible.parsing.metadata import extract_metadata from ansible.parsing.plugin_docs import read_docstub from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.plugins.loader import action_loader, fragment_loader -from ansible.utils.collection_loader import set_collection_playbook_paths +from ansible.utils.collection_loader import set_collection_playbook_paths, list_collection_dirs, get_collection_name_from_path from ansible.utils.display import Display from ansible.utils.plugin_docs import BLACKLIST, get_docstring, get_versioned_doclink display = Display() @@ -36,6 +36,17 @@ def jdump(text): display.display(json.dumps(text, sort_keys=True, indent=4)) +def add_collection_plugins(plugin_list, plugin_type): + + colldirs = list_collection_dirs() + for ns in colldirs.keys(): + for path in colldirs[ns]: + + collname = get_collection_name_from_path(path) + ptype = C.COLLECTION_PTYPE_COMPAT.get(plugin_type, plugin_type) + plugin_list.update(DocCLI.find_plugins(os.path.join(path, 'plugins', ptype), plugin_type, collname)) + + class RemovedPlugin(Exception): pass @@ -125,6 +136,8 @@ class DocCLI(CLI): for path in paths: self.plugin_list.update(DocCLI.find_plugins(path, plugin_type)) + add_collection_plugins(self.plugin_list, plugin_type) + plugins = self._get_plugin_list_filenames(loader) if do_json: jdump(plugins) @@ -146,6 +159,8 @@ class DocCLI(CLI): for path in paths: self.plugin_list.update(DocCLI.find_plugins(path, plugin_type)) + add_collection_plugins(self.plugin_list, plugin_type) + descs = self._get_plugin_list_descriptions(loader) if do_json: jdump(descs) @@ -349,7 +364,7 @@ class DocCLI(CLI): return text @staticmethod - def find_plugins(path, ptype): + def find_plugins(path, ptype, collection=None): display.vvvv("Searching %s for plugins" % path) @@ -386,6 +401,10 @@ class DocCLI(CLI): plugin = plugin.lstrip('_') # remove underscore from deprecated plugins if plugin not in BLACKLIST.get(bkey, ()): + + if collection: + plugin = '%s.%s' % (collection, plugin) + plugin_list.add(plugin) display.vvvv("Added %s" % plugin) diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index 754145d115b..d94fabc1962 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -97,6 +97,7 @@ BECOME_METHODS = _DeprecatedSequenceConstant( # CONSTANTS ### yes, actual ones BLACKLIST_EXTS = ('.pyc', '.pyo', '.swp', '.bak', '~', '.rpm', '.md', '.txt', '.rst') BOOL_TRUE = BOOLEANS_TRUE +COLLECTION_PTYPE_COMPAT = {'module': 'modules'} DEFAULT_BECOME_PASS = None DEFAULT_PASSWORD_CHARS = to_text(ascii_letters + digits + ".,:-_", errors='strict') # characters included in auto-generated passwords DEFAULT_REMOTE_PASS = None diff --git a/lib/ansible/utils/collection_loader.py b/lib/ansible/utils/collection_loader.py index bf4d7589480..21c8384b2a3 100644 --- a/lib/ansible/utils/collection_loader.py +++ b/lib/ansible/utils/collection_loader.py @@ -6,17 +6,21 @@ __metaclass__ = type import os import os.path -import pkgutil import re import sys from types import ModuleType +from collections import defaultdict +from ansible import constants as C +from ansible.utils.display import Display from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils.compat.importlib import import_module from ansible.module_utils.six import iteritems, string_types, with_metaclass from ansible.utils.singleton import Singleton +display = Display() + _SYNTHETIC_PACKAGES = { # these provide fallback package definitions when there are no on-disk paths 'ansible_collections': dict(type='pkg_only', allow_external_subpackages=True), @@ -28,6 +32,8 @@ _SYNTHETIC_PACKAGES = { 'ansible_collections.ansible.builtin.plugins.modules': dict(type='flatmap', flatmap='ansible.modules', graft=True), } +FLAG_FILES = frozenset(['MANIFEST.json', 'galaxy.yml']) + # FIXME: exception handling/error logging class AnsibleCollectionLoader(with_metaclass(Singleton, object)): @@ -588,3 +594,60 @@ def get_collection_name_from_path(path): def set_collection_playbook_paths(b_playbook_paths): AnsibleCollectionLoader().set_playbook_paths(b_playbook_paths) + + +def is_collection_path(path): + + if os.path.isdir(path): + for flag in FLAG_FILES: + if os.path.exists(os.path.join(path, flag)): + return True + + return False + + +def list_valid_collection_paths(search_paths=None, warn=False): + + found_paths = [] + if search_paths is None: + search_paths = C.COLLECTIONS_PATHS + + for path in search_paths: + + if not os.path.exists(path): + # warn for missing, but not if default + if warn: + display.warning("The configured collection path {0} does not exist.".format(path)) + continue + + if not os.path.isdir(path): + if warn: + display.warning("The configured collection path {0}, exists, but it is not a directory.".format(path)) + continue + + found_paths.append(path) + + return found_paths + + +def list_collection_dirs(search_paths=None, namespace=None): + + collections = defaultdict(list) + paths = list_valid_collection_paths(search_paths) + for path in paths: + + if os.path.isdir(path): + coll_root = os.path.join(path, 'ansible_collections') + + if os.path.exists(coll_root) and os.path.isdir(coll_root): + for namespace in os.listdir(coll_root): + namespace_dir = os.path.join(coll_root, namespace) + + if os.path.isdir(namespace_dir): + for collection in os.listdir(namespace_dir): + coll_dir = os.path.join(namespace_dir, collection) + + if is_collection_path(coll_dir): + collections[namespace].append(os.path.join(namespace_dir, collection)) + + return collections