Look for globally installed collection plugins

Use mazer installed plugins
Merge branch 'devel' into mazer_role_loader
2018-09-12 13:05:27 -04:00 · 2018-09-06 16:24:10 -04:00 · 2018-08-27 15:46:18 -04:00 · 2018-08-10 10:29:18 -04:00 · 2018-08-08 18:19:24 -04:00 · 2018-07-27 12:40:36 -04:00
9 changed files with 520 additions and 20 deletions
--- a/lib/ansible/cli/init.py
+++ b/lib/ansible/cli/init.py
@ -623,7 +623,10 @@ class CLI(with_metaclass(ABCMeta, object)):
            cpath = "Default w/o overrides"
        else:
            cpath = C.DEFAULT_MODULE_PATH
+        conpath = C.DEFAULT_CONTENT_PATH or "Default w/o overrides"
+
        result = result + "\n  configured module search path = %s" % cpath
+        result = result + "\n  configured galaxy content search path = %s" % conpath
        result = result + "\n  ansible python module location = %s" % ':'.join(ansible.__path__)
        result = result + "\n  executable location = %s" % sys.argv[0]
        result = result + "\n  python version = %s" % ''.join(sys.version.splitlines())
--- a/lib/ansible/config/base.yml
+++ b/lib/ansible/config/base.yml
@ -343,7 +343,7 @@ LOCALHOST_WARNING:
  version_added: "2.6"
 DEFAULT_ACTION_PLUGIN_PATH:
  name: Action plugins path
-  default: ~/.ansible/plugins/action:/usr/share/ansible/plugins/action
+  default: ~/.ansible/content/*/*/plugins/actions:~/.ansible/plugins/action:/usr/share/ansible/content/*/*/plugins/action:/usr/share/ansible/plugins/action
  description: Colon separated paths in which Ansible will search for Action Plugins.
  env: [{name: ANSIBLE_ACTION_PLUGINS}]
  ini:
@ -475,7 +475,7 @@ DEFAULT_CALLABLE_WHITELIST:
  type: list
 DEFAULT_CALLBACK_PLUGIN_PATH:
  name: Callback Plugins Path
-  default: ~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback
+  default: ~/.ansible/content/*/*/plugins/callback:~/.ansible/plugins/callback:/usr/share/ansible/content/*/*/plugins/callback:/usr/share/ansible/plugins/callback
  description: Colon separated paths in which Ansible will search for Callback Plugins.
  env: [{name: ANSIBLE_CALLBACK_PLUGINS}]
  ini:
@ -495,7 +495,7 @@ DEFAULT_CALLBACK_WHITELIST:
  yaml: {key: plugins.callback.whitelist}
 DEFAULT_CLICONF_PLUGIN_PATH:
  name: Cliconf Plugins Path
-  default: ~/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf
+  default: ~/.ansible/content/*/*/plugins/cliconf:~/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf
  description: Colon separated paths in which Ansible will search for Cliconf Plugins.
  env: [{name: ANSIBLE_CLICONF_PLUGINS}]
  ini:
@ -503,13 +503,21 @@ DEFAULT_CLICONF_PLUGIN_PATH:
  type: pathspec
 DEFAULT_CONNECTION_PLUGIN_PATH:
  name: Connection Plugins Path
-  default: ~/.ansible/plugins/connection:/usr/share/ansible/plugins/connection
+  default: ~/.ansible/content/*/*/plugins/connection:~/.ansible/plugins/connection:/usr/share/ansible/content/*/*/plugins/connection:/usr/share/ansible/plugins/connection
  description: Colon separated paths in which Ansible will search for Connection Plugins.
  env: [{name: ANSIBLE_CONNECTION_PLUGINS}]
  ini:
  - {key: connection_plugins, section: defaults}
  type: pathspec
  yaml: {key: plugins.connection.path}
+DEFAULT_CONTENT_PATH:
+  name: Ansible Galaxy Content Path
+  description: Colon separated paths in which Ansible will search for Roles, Modules and Plugins installed as Galaxy Content.
+  default: ~/.ansible/content
+  env: [{name: ANSIBLE_CONTENT_PATH}]
+  ini:
+  - {key: content_path, section: defaults}
+  type: pathspec
 DEFAULT_DEBUG:
  name: Debug mode
  default: False
@ -545,7 +553,7 @@ DEFAULT_FACT_PATH:
  yaml: {key: facts.gathering.fact_path}
 DEFAULT_FILTER_PLUGIN_PATH:
  name: Jinja2 Filter Plugins Path
-  default: ~/.ansible/plugins/filter:/usr/share/ansible/plugins/filter
+  default: ~/.ansible/content/*/*/plugins/filter:~/.ansible/plugins/filter:/usr/share/ansible/content/*/*/plugins/action:/usr/share/ansible/plugins/filter
  description: Colon separated paths in which Ansible will search for Jinja2 Filter Plugins.
  env: [{name: ANSIBLE_FILTER_PLUGINS}]
  ini:
@ -665,7 +673,7 @@ DEFAULT_HOST_LIST:
  yaml: {key: defaults.inventory}
 DEFAULT_HTTPAPI_PLUGIN_PATH:
  name: HttpApi Plugins Path
-  default: ~/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi
+  default: ~/.ansible/content/*/*/plugins/httpapi:~/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi
  description: Colon separated paths in which Ansible will search for HttpApi Plugins.
  env: [{name: ANSIBLE_HTTPAPI_PLUGINS}]
  ini:
@ -687,7 +695,7 @@ DEFAULT_INTERNAL_POLL_INTERVAL:
    - "The default corresponds to the value hardcoded in Ansible <= 2.1"
 DEFAULT_INVENTORY_PLUGIN_PATH:
  name: Inventory Plugins Path
-  default: ~/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory
+  default: ~/.ansible/content/*/*/plugins/inventory:~/.ansible/plugins/inventory:/usr/share/ansible/content/*/*/plugins/inventory:/usr/share/ansible/plugins/inventory
  description: Colon separated paths in which Ansible will search for Inventory Plugins.
  env: [{name: ANSIBLE_INVENTORY_PLUGINS}]
  ini:
@ -771,7 +779,7 @@ DEFAULT_LOG_FILTER:
 DEFAULT_LOOKUP_PLUGIN_PATH:
  name: Lookup Plugins Path
  description: Colon separated paths in which Ansible will search for Lookup Plugins.
-  default: ~/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup
+  default: ~/.ansible/content/*/*/plugins/lookup:~/.ansible/plugins/lookup:/usr/share/ansible/content/*/*/plugins/lookup:/usr/share/ansible/plugins/lookup
  env: [{name: ANSIBLE_LOOKUP_PLUGINS}]
  ini:
  - {key: lookup_plugins, section: defaults}
@ -825,7 +833,7 @@ DEFAULT_MODULE_NAME:
 DEFAULT_MODULE_PATH:
  name: Modules Path
  description: Colon separated paths in which Ansible will search for Modules.
-  default: ~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules
+  default: ~/.ansible/content/*/*/modules:~/.ansible/plugins/modules:/usr/share/ansible/content/*/*/modules:/usr/share/ansible/plugins/modules
  env: [{name: ANSIBLE_LIBRARY}]
  ini:
  - {key: library, section: defaults}
@ -845,14 +853,14 @@ DEFAULT_MODULE_SET_LOCALE:
 DEFAULT_MODULE_UTILS_PATH:
  name: Module Utils Path
  description: Colon separated paths in which Ansible will search for Module utils files, which are shared by modules.
-  default: ~/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils
+  default: ~/.ansible/content/*/*/module_utils:~/.ansible/plugins/module_utils:/usr/share/ansible/content/*/*/module_utils:/usr/share/ansible/plugins/module_utils
  env: [{name: ANSIBLE_MODULE_UTILS}]
  ini:
  - {key: module_utils, section: defaults}
  type: pathspec
 DEFAULT_NETCONF_PLUGIN_PATH:
  name: Netconf Plugins Path
-  default: ~/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf
+  default: ~/.ansible/content/*/*/netconf_plugins:~/.ansible/plugins/netconf:/usr/share/ansible/content/*/*/plugins/netconf:/usr/share/ansible/plugins/netconf
  description: Colon separated paths in which Ansible will search for Netconf Plugins.
  env: [{name: ANSIBLE_NETCONF_PLUGINS}]
  ini:
@ -937,7 +945,7 @@ DEFAULT_REMOTE_USER:
  - {key: remote_user, section: defaults}
 DEFAULT_ROLES_PATH:
  name: Roles path
-  default: ~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles
+  default: ~/ansible/content/*/*/roles:~/.ansible/roles:/usr/share/ansible/content/*/*/roles:/usr/share/ansible/roles:/etc/ansible/roles
  description: Colon separated paths in which Ansible will search for Roles.
  env: [{name: ANSIBLE_ROLES_PATH}]
  expand_relative_paths: True
@ -1046,7 +1054,7 @@ DEFAULT_STRATEGY:
 DEFAULT_STRATEGY_PLUGIN_PATH:
  name: Strategy Plugins Path
  description: Colon separated paths in which Ansible will search for Strategy Plugins.
-  default: ~/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy
+  default: ~/.ansible/content/*/*/plugins/strategy:~/.ansible/plugins/strategy:/usr/share/ansible/content/*/*/plugins/strategy:/usr/share/ansible/plugins/strategy
  env: [{name: ANSIBLE_STRATEGY_PLUGINS}]
  ini:
  - {key: strategy_plugins, section: defaults}
@ -1159,7 +1167,7 @@ DEFAULT_TASK_INCLUDES_STATIC:
    alternatives: None, as its already built into the decision between include_tasks and import_tasks
 DEFAULT_TERMINAL_PLUGIN_PATH:
  name: Terminal Plugins Path
-  default: ~/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal
+  default: ~/.ansible/content/*/*/plugins/terminal:~/.ansible/plugins/terminal:/usr/share/ansible/content/*/*/plugins/terminal:/usr/share/ansible/plugins/terminal
  description: Colon separated paths in which Ansible will search for Terminal Plugins.
  env: [{name: ANSIBLE_TERMINAL_PLUGINS}]
  ini:
@ -1168,7 +1176,7 @@ DEFAULT_TERMINAL_PLUGIN_PATH:
 DEFAULT_TEST_PLUGIN_PATH:
  name: Jinja2 Test Plugins Path
  description: Colon separated paths in which Ansible will search for Jinja2 Test Plugins.
-  default: ~/.ansible/plugins/test:/usr/share/ansible/plugins/test
+  default: ~/.ansible/content/*/*/plugins/test:~/.ansible/plugins/test:/usr/share/ansible/content/*/*/plugins/test:/usr/share/ansible/plugins/test
  env: [{name: ANSIBLE_TEST_PLUGINS}]
  ini:
  - {key: test_plugins, section: defaults}
@ -1201,7 +1209,7 @@ DEFAULT_UNDEFINED_VAR_BEHAVIOR:
  type: boolean
 DEFAULT_VARS_PLUGIN_PATH:
  name: Vars Plugins Path
-  default: ~/.ansible/plugins/vars:/usr/share/ansible/plugins/vars
+  default: ~/.ansible/content/*/*/plugins/vars:~/.ansible/plugins/vars:/usr/share/ansible/content/*/*/plugins/vars:/usr/share/ansible/plugins/vars
  description: Colon separated paths in which Ansible will search for Vars Plugins.
  env: [{name: ANSIBLE_VARS_PLUGINS}]
  ini:
--- a/lib/ansible/playbook/role/definition.py
+++ b/lib/ansible/playbook/role/definition.py
@ -20,12 +20,14 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

 import os
+import pprint

-from ansible import constants as C
+from ansible import constants as C  # noqa
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, string_types
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
-from ansible.playbook.attribute import Attribute, FieldAttribute
+# from ansible.playbook.attribute import Attribute
+from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
 from ansible.playbook.conditional import Conditional
@ -33,6 +35,9 @@ from ansible.playbook.taggable import Taggable
 from ansible.template import Templar
 from ansible.utils.path import unfrackpath

+import logging
+log = logging.getLogger(__name__)
+
 try:
    from __main__ import display
 except ImportError:
@ -43,6 +48,112 @@ except ImportError:
 __all__ = ['RoleDefinition']


+def role_name_to_relative_content_path(role_name):
+    '''Translate a namespace.reponame.rolename to relative content path.
+
+    ie, testing.some_repo.some_role -> testing/some_repo/roles/some_role
+    '''
+
+    content_rel_role_path = None
+
+    # TODO: decide if something like 'geerlingguy.nginx' is sufficient
+    #       ie, if there will be a short names/aliases/default resolvers
+    try:
+        # namespace, repo, name = role_name.split('.', 2)
+        name_parts = role_name.split('.', 2)
+    except (ValueError, AttributeError):
+        log.debug('Could not "." split role_name "%s"',
+                  role_name)
+        return None
+
+    # name_parts namespace, repo, rolename or
+    #  namespace.reponame (and assume there is a also a role named 'reponame'
+    #  at namespace/repo/roles/reponame. ie, old style roles with one role
+    # in a repo.)
+    # geerlingguy.apache -> geerlingguy/apache/roles/apache
+    # geerlingguy.apache.apache -> geerlingguy/apache/roles/apache
+    # testing.multi.apache -> testing/multi/roles/apache
+    # testing.multi ->  testing/multi/roles/multi (if it exists)
+    # WARNING: if a repo name for a multicontent repo matches a role name
+    #          coincedently, this introduces an ambiquity
+    # mynamespace.install.add_user -> mynamespace/install/roles/add_user
+    # mynamespace.install.install -> mynamespace/install/roles/install
+    # mynamespace.install -> mynamespace/install/roles/install
+
+    if len(name_parts) < 2:
+        return None
+
+    log.debug(name_parts)
+
+    # catches 'namespace.' (trailing dot) as well as rel path cases
+    # like '../some_dir.foo.bar'
+    if not all(name_parts):
+        return None
+
+    log.debug('name_parts: %s', name_parts)
+
+    namespace = name_parts.pop(0)
+    repo = name_parts.pop(0)
+
+    name = None
+    try:
+        name = name_parts.pop(0)
+    except IndexError:
+        log.debug('role name "%s" only had two name parts (namespace="%s", repo="%s") and no role name',
+                  role_name, namespace, repo)
+
+    if name:
+        content_rel_role_path = os.path.join(namespace, repo, 'roles', name)
+    else:
+        content_rel_role_path = os.path.join(namespace, repo, 'roles', repo)
+
+    return content_rel_role_path
+
+
+def find_role_in_content_path(role_name, loader, content_search_paths):
+    '''search for role in ~/.ansible/content and return first match.
+
+    return None if no matches'''
+
+    # try the galaxy content paths
+    # TODO: this is where a 'role spec resolver' could be plugged into.
+    #       The resolver would be responsible parsing/understanding the role spec
+    #       (a formatted string or a dict), and figuring out the approriate galaxy
+    #       namespace, repo name, and role name.
+    #
+    #       The next step would be finding that role on the fs.
+    #       If there are conflicts or ambiquity, the resolver would apply
+    #       any rules or convention or precedence to choose the correct role.
+    #       For ex, if namespace isnt provided, and 2 or more namespaces have a
+    #       role that matches, the resolver would choose.
+    # FIXME: mv to method, deindent, return early, etc
+
+    log.debug('content_search_paths: %s', content_search_paths)
+
+    content_rel_role_path = role_name_to_relative_content_path(role_name)
+
+    log.debug('content_rel_role_path: %s', content_rel_role_path)
+
+    # didn't parse the role_name, return None
+    if not content_rel_role_path:
+        return None
+
+    # TODO: the for loop isnt needed if we really really only
+    #       support one content path
+    for content_search_path in content_search_paths:
+
+        fq_role_path = os.path.join(content_search_path, content_rel_role_path)
+        fq_role_path = unfrackpath(fq_role_path)
+
+        log.debug('fq_role_path: %s', fq_role_path)
+
+        if loader.path_exists(fq_role_path):
+            log.info('FOUND:     %s at content path "%s"', role_name, fq_role_path)
+            return (role_name, fq_role_path)
+
+    return None
+
+
 class RoleDefinition(Base, Become, Conditional, Taggable):

    _role = FieldAttribute(isa='string')
@ -119,6 +230,7 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
        '''

        if isinstance(ds, string_types):
+            log.debug('role_name: %s (role ds was a string)', ds)
            return ds

        role_name = ds.get('role', ds.get('name'))
@ -133,6 +245,8 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
            if templar._contains_vars(role_name):
                role_name = templar.template(role_name)

+        log.info('using role_name: %s', role_name)
+        log.info('role_name: %s ds:\n%s', role_name, pprint.pformat(ds))
        return role_name

    def _load_role_path(self, role_name):
@ -143,6 +257,9 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
        append it to the default role path
        '''

+        log.info('Look for:  %s', role_name)
+        # log.debug('installed_role_spec: %s', installed_role_spec)
+
        # we always start the search for roles in the base directory of the playbook
        role_search_paths = [
            os.path.join(self._loader.get_basedir(), u'roles'),
@ -161,6 +278,7 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
        # in the loader (which should be the playbook dir itself) but without
        # the roles/ dir appended
        role_search_paths.append(self._loader.get_basedir())
+        log.debug('role_search_paths: %s', role_search_paths)

        # create a templar class to template the dependency names, in
        # case they contain variables
@ -172,19 +290,46 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
        templar = Templar(loader=self._loader, variables=all_vars)
        role_name = templar.template(role_name)

+        # Look for roles in the content search path (~/.ansible/content) based on dotted role
+        # names.
+        content_results = find_role_in_content_path(role_name, self._loader,
+                                                    content_search_paths=C.DEFAULT_CONTENT_PATH)
+        log.debug('content_results: %s', content_results)
+
+        if content_results:
+            log.debug('returning %s (found a role in content path for role_name=%s)',
+                      repr(content_results), role_name)
+
+            return content_results
+
        # now iterate through the possible paths and return the first one we find
        for path in role_search_paths:
            path = templar.template(path)
+
+            # fq_role_name = resolve_role_name(role_name)
            role_path = unfrackpath(os.path.join(path, role_name))
+
+            log.debug('search for role=%s in path: %s (role_path=%s)', role_name, path, role_path)
            if self._loader.path_exists(role_path):
+                log.info('FOUND:     %s at role path: "%s"', role_name, role_path)
+
                return (role_name, role_path)

        # if not found elsewhere try to extract path from name
        role_path = unfrackpath(role_name)
+
+        log.debug('trying role_name=%s as a path: %s ', role_name, role_path)
+
        if self._loader.path_exists(role_path):
+            log.debug('FOUND:     %s (via relative path) at path: "%s"', role_name, role_path)
+
            role_name = os.path.basename(role_name)
            return (role_name, role_path)

+        log.info('Failed to find the role "%s" in content paths %s', role_name, C.DEFAULT_CONTENT_PATH)
+        log.info('Failed to find the role "%s" in any of the roles paths: %s',
+                 role_name, ":".join(role_search_paths))
+
        raise AnsibleError("the role '%s' was not found in %s" % (role_name, ":".join(role_search_paths)), obj=self._ds)

    def _split_role_params(self, ds):
--- a/lib/ansible/playbook/role/requirement.py
+++ b/lib/ansible/playbook/role/requirement.py
@ -112,7 +112,7 @@ class RoleRequirement(RoleDefinition):
        else:
            role = role.copy()

-            if 'src'in role:
+            if 'src' in role:
                # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
                if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'):
                    role["src"] = "git+" + role["src"]
--- a/lib/ansible/plugins/loader.py
+++ b/lib/ansible/plugins/loader.py
@ -58,7 +58,7 @@ class PluginLoader:
        elif not config:
            config = []

-        self.config = config
+        self.config = self.expand_path_globs(config)

        if class_name not in MODULE_CACHE:
            MODULE_CACHE[class_name] = {}
@ -111,6 +111,19 @@ class PluginLoader:
            PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name],
        )

+    def expand_path_globs(self, paths):
+        generated = []
+        for path in paths:
+            if 'content/*' not in path:
+                generated.append(path)
+                continue
+
+            dirs = glob.glob(path)
+            if dirs:
+                generated += dirs
+
+        return generated
+
    def format_paths(self, paths):
        ''' Returns a string suitable for printing of the search path '''

@ -267,6 +280,7 @@ class PluginLoader:
        #       looks like _get_paths() never forces a cache refresh so if we expect
        #       additional directories to be added later, it is buggy.
        for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)):
+
            try:
                full_paths = (os.path.join(path, f) for f in os.listdir(path))
            except OSError as e:
@ -301,6 +315,28 @@ class PluginLoader:
                if full_name not in self._plugin_path_cache[extension]:
                    self._plugin_path_cache[extension][full_name] = full_path

+                # MAZER
+                if '/content/' in path:
+                    path_elements = path.split(os.path.sep)
+                    content_index = path_elements.index('content')
+                    user_namespace = path_elements[content_index + 1]
+                    repository_name = path_elements[content_index + 2]
+
+                    pyfqn = (user_namespace + '.' + repository_name + '.' + base_name).replace('-', '_')
+                    fqn = user_namespace + '.' + repository_name + '.' + base_name
+                    pyrqn = (repository_name + '.' + base_name).replace('-', '_')
+                    rqn = repository_name + '.' + base_name
+
+                    for ext in ['', extension]:
+                        for qn in [pyfqn, fqn, pyrqn, rqn]:
+                            self._plugin_path_cache[ext][qn] = full_path
+
+                            # not sure why this is necessary for lookups
+                            pull_cache[qn] = full_path
+                            if full_path.endswith('.py'):
+                                self._plugin_path_cache[ext][qn + '.py'] = full_path
+                                pull_cache[qn + '.py'] = full_path
+
            self._searched_paths.add(path)
            try:
                return pull_cache[name]
@ -523,6 +559,29 @@ class PluginLoader:
                self._load_config_defs(basename, path)

            self._update_object(obj, basename, path)
+
+            # MAZER
+            if hasattr(obj, 'filters') and '/content/' in path:
+                path_elements = path.split(os.path.sep)
+                content_index = path_elements.index('content')
+                namespace = path_elements[content_index + 1]
+                repository = path_elements[content_index + 2]
+
+                filter_dict = obj.filters()
+                for key in filter_dict.keys():
+                    newkeys = [
+                        namespace + '.' + repository + '.' + key,
+                        repository + '.' + key
+                    ]
+                    for nk in newkeys:
+                        nk = nk.replace('-', '_')
+                        filter_dict[nk] = filter_dict[key]
+
+                def patch_filters():
+                    return  filter_dict
+
+                obj.filters = patch_filters
+
            yield obj


--- a/test/units/playbook/role/test_definition.py
+++ b/test/units/playbook/role/test_definition.py
@ -0,0 +1,153 @@
+import logging
+import os.path
+
+import pytest
+
+from ansible import errors
+from ansible.playbook.role import definition
+
+log = logging.getLogger(__name__)
+
+
+def test_role_definition(mocker):
+    res = definition.RoleDefinition(play=None, role_basedir=None, variable_manager=None, loader=None)
+
+    assert isinstance(res, definition.RoleDefinition)
+
+
+def test_role_definition_load_role_path_from_content_path(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/roles')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    rd = definition.RoleDefinition(play=None, role_basedir=None, variable_manager=None, loader=mock_loader)
+
+    role_name = 'namespace.repo.role'
+    res = rd._load_role_path(role_name)
+
+    log.debug('res: %s', res)
+    assert isinstance(res, tuple)
+    assert res[0] == role_name
+    assert res[1] == os.path.join(content_path, 'namespace/repo/roles/role')
+
+
+def test_role_definition_load_role_path_from_role_path(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    rd.preprocess_data({'role': role_name})
+    res = rd._load_role_path(role_name)
+
+    log.debug('res: %s', res)
+    assert isinstance(res, tuple)
+    # The playbook rel roles/ dir is the first path checked
+    assert res[1] == '/dev/null/playbook_rel_roles_dir/roles/some_role'
+
+
+def test_role_definition_load_role_path_from_role_path_not_found(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=False)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    with pytest.raises(errors.AnsibleError, match='.'):
+        # pres = rd.preprocess_data({'role': role_name})
+        pres = rd.preprocess_data(role_name)
+        log.debug('pres: %s', pres)
+        rd._load_role_path(role_name)
+
+
+def test_role_definition_load_role_no_name_in_role_ds(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    with pytest.raises(errors.AnsibleError, match='role definitions must contain a role name'):
+        rd.preprocess_data({'stuff': role_name,
+                            'things_i_like': ['cheese', 'naps']})
+
+
+@pytest.mark.parametrize("role_name,expected",
+                         [('namespace.repo.role', 'namespace/repo/roles/role'),
+                          ('a.a.a', 'a/a/roles/a'),
+                          ('too.many.dots.lets.assume.extra.dots.are.role.name',
+                           'too/many/roles/dots.lets.assume.extra.dots.are.role.name'),
+                          # valid if role names can include sub paths? should raise an error?
+                          ('ns.repo.role/name/roles/role', 'ns/repo/roles/role/name/roles/role'),
+                          # DWIM
+                          ('geerlingguy.apache', 'geerlingguy/apache/roles/apache'),
+
+                          # these are actually legit 'role_name' inside ansible
+                          # they get used as relative paths
+                          ('../../some_dir/foo', None),
+                          ('../some_dir.foo.bar', None),
+                          ('.', None),
+                          ('/', None),
+                          ('./foo', None),
+                          ('.', None),
+
+                          ('justaname', None),
+                          ('namespace_dot.', None),
+                          ('somenamespace/somerepo/roles/somerole', None),
+
+                          (None, None),
+                          ])
+def test_role_name_to_relative_content_path(role_name, expected):
+    res = definition.role_name_to_relative_content_path(role_name)
+    log.debug('res: %s', res)
+    assert res == expected
+
+
+def test_find_role_in_content_path_invalid_role_name(mocker):
+    mock_loader = mocker.Mock(return_value=False)
+    res = definition.find_role_in_content_path('justaname', mock_loader, '/dev/null/foo')
+
+    assert res is None
+
+
+def test_find_role_in_content_path_loader_cant_find(mocker):
+    mock_loader = mocker.Mock(path_exists=mocker.Mock(return_value=False))
+    res = definition.find_role_in_content_path('namespace.repo.rolename', mock_loader, ['/dev/null/foo'])
+
+    assert res is None
+
+
+def test_find_role_in_content_path_loader(mocker):
+    content_search_path = '/dev/null/foo'
+    role_name = 'namespace.repo.rolename'
+    mock_loader = mocker.Mock(path_exists=mocker.Mock(return_value=True))
+    res = definition.find_role_in_content_path(role_name, mock_loader, [content_search_path])
+
+    assert isinstance(res, tuple)
+    assert res[0] == role_name
+    assert res[1] == os.path.join(content_search_path, 'namespace/repo/roles/rolename')
--- a/test/units/playbook/role/test_metadata.py
+++ b/test/units/playbook/role/test_metadata.py
@ -0,0 +1,13 @@
+import logging
+
+from ansible.playbook.role import metadata
+
+log = logging.getLogger(__name__)
+
+
+def test_role_metadata():
+    rmd = metadata.RoleMetadata()
+
+    log.debug('rmd: %s', rmd)
+
+    assert isinstance(rmd, metadata.RoleMetadata)
--- a/test/units/playbook/role/test_requirement.py
+++ b/test/units/playbook/role/test_requirement.py
@ -0,0 +1,87 @@
+import logging
+
+from ansible.playbook.role import definition
+from ansible.playbook.role import requirement
+
+log = logging.getLogger(__name__)
+
+
+def test_role_requirement():
+    rr = requirement.RoleRequirement()
+
+    assert isinstance(rr, requirement.RoleRequirement)
+    assert isinstance(rr, definition.RoleDefinition)
+
+
+def test_repo_url_to_repo_name():
+    repo_url = 'http://git.example.com/repos/repo.git'
+    res = requirement.RoleRequirement.repo_url_to_role_name(repo_url)
+
+    log.debug('res: %s', res)
+
+    assert res == 'repo'
+
+
+def test_role_spec_parse():
+
+    res = requirement.RoleRequirement.role_spec_parse('foo.bar')
+
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'foo.bar'
+    assert res['src'] == 'foo.bar'
+
+
+def test_role_spec_parse_example():
+    role_spec = 'git+http://git.example.com/repos/repo.git,v1.0'
+    res = requirement.RoleRequirement.role_spec_parse(role_spec)
+
+    # {'scm': 'git', 'src': 'http://git.example.com/repos/repo.git',
+    #  'version': 'v1.0', 'name': 'repo'}
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'repo'
+    assert res['scm'] == 'git'
+    assert res['src'] == 'http://git.example.com/repos/repo.git'
+    assert res['version'] == 'v1.0'
+
+
+def test_role_yaml_parse_dict_old_style():
+    role_yaml = {'role': "galaxy.role,v1.2.3,role_name", 'other_vars': "here"}
+    res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+
+    log.debug('res: %s', res)
+    # {'scm': None, 'src': 'galaxy.role', 'version': 'v1.2.3', 'name': 'role_name'}
+    assert isinstance(res, dict)
+    assert res['name'] == 'role_name'
+    assert res['version'] == 'v1.2.3'
+    assert res['src'] == 'galaxy.role'
+
+#
+# def test_role_yaml_parse_dict_new_style():
+#     # NOTE: the example in the comments for role_yaml_parse fails
+#     role_yaml = {'src': 'git+http://github.com/some_galaxy/some_role,v1.2.3,role_name', 'other_vars': "here"}
+#
+#     res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+#
+#     log.debug('res: %s', res)
+#     # {'scm': None, 'src': 'galaxy.role', 'version': 'v1.2.3', 'name': 'role_name'}
+#     assert isinstance(res, dict)
+#     assert res['name'] == 'role_name'
+#     assert res['version'] == 'v1.2.3'
+#     assert res['src'] == 'galaxy.role'
+#
+
+
+def test_role_yaml_parse_string():
+    role_yaml = 'galaxy.role,v1.2.3,role_name'
+    res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'role_name'
+    assert res['version'] == 'v1.2.3'
+    assert res['src'] == 'galaxy.role'
--- a/test/units/playbook/role/test_role.py
+++ b/test/units/playbook/role/test_role.py
@ -20,6 +20,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

 import collections
+import logging

 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
@ -35,6 +36,8 @@ from ansible.playbook.role import Role
 from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role import hash_params

+log = logging.getLogger(__name__)
+

 class TestHashParams(unittest.TestCase):
    def test(self):
@ -378,3 +381,32 @@ class TestRole(unittest.TestCase):
        r = Role.load(i, play=mock_play)

        self.assertEqual(r.get_name(), "foo_complex")
+
+    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
+    def test_serialize(self):
+
+        fake_loader = DictDataLoader({
+            "/etc/ansible/roles/foo_vars/defaults/main/foo/bar.yml": """
+            foo: bar
+            """,
+            "/etc/ansible/roles/foo_vars/vars/main/bar/foo.yml": """
+            foo: bam
+            """,
+        })
+
+        mock_play = MagicMock()
+        mock_play.ROLE_CACHE = {}
+
+        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
+        r = Role.load(i, play=mock_play)
+
+        res = r.serialize()
+        log.debug('res: %s', res)
+
+        self.assertEqual(r._default_vars, dict(foo='bar'))
+        self.assertEqual(r._role_vars, dict(foo='bam'))
+
+        r2 = Role.load(i, play=mock_play)
+        r2.deserialize(res)
+
+        log.debug('r2: %s', r2)
Author	SHA1	Message	Date
James Tanner	565d4639b3	Look for globally installed collection plugins	2018-09-12 13:05:27 -04:00
James Tanner	2b228874f4	Use mazer installed plugins	2018-09-06 16:24:10 -04:00
Adrian Likins	1f55cf4ed4	Merge branch 'devel' into mazer_role_loader * devel: (513 commits) Fix systemd service is already masked issue (#44730) fix issue with no_log in py3 modules/terraform: Quote the variable values in the command line (#43493) YUM4/DNF compatibility via yum action plugin (#44322) BOTMETA.yml: remove superfluous labels (#44628) Share the implementation of hashing for both vars_prompt and password_hash (#21215) one_host environment variables, Fixes #44163 (#44568) ec2: add "IAM Role" to instance_profile_name ios_vrf speed fix (#43765) fix typo (#44712) junos cli_config idempotence fix (#44706) Switch to LiteralPath instead of Path. Closes #44508 (#44509) Module win_domain_computer fix delete computer with child (#44500) ACME: improve documentation (#44691) doc: fixed typo (#44685) IPA: Add option to specify timeout (#44572) Added nios_txt_record module (#39264) adds the bigip_cli_script module (#44674) Clean up BOTMETA.yml (#44574) Change validate-modules for removed modules ...	2018-08-27 15:46:18 -04:00
Adrian Likins	1431900da2	Merge branch 'devel' into mazer_role_loader * devel: (30 commits) Prevent data being truncated over persistent connection socket (#43885) Fix eos_command integration test failures (#43922) Update iosxr cliconf plugin (#43837) win_domain modules: ensure Netlogon service is still running after promotion (#43703) openvswitch_db : Handle column value conversion and idempotency in no_key case (#43869) Fix typo Fix spelling of ansbile to ansible (#43898) added platform guides for NOS and VOSS (#43854) Fix download URL for yum integration test. New module for managing EMC VNX Block storage (#42945) Docker integration tests: factorize setup (#42306) VMware: datastore selection (#35812) Remove unnecessary features from cli_command (#43829) [doc] import_role: mention version from which behavior changed and fix some typos (#43843) Add source interface and use-vrf features (#43418) Fix unreferenced msg from vmware_host (#43872) set supports_generate_diff to False vyos (#43873) add group_by_os_family in azure dynamic inventory (#40702) ansible-test: Create public key creating Windows targets (#43760) azure_rm_loadbalancer_facts.py: list() takes at least 2 arguments fix (#29046) (#29050) ...	2018-08-10 10:29:18 -04:00
Adrian Likins	545da55b88	Merge branch 'devel' into mazer_role_loader * devel: (177 commits) Fix all ACI examples to use delegate_to (#43439) Ensure removed_in is StrictVersion before comparing (#43835) Network *_config action plugin updates (#43838) Lenovo size reduce cnos.py of util (#43823) Add OpenStack Magnum cluster template module (#42654) Correct task args in azure_rm_managed_disk test. Fix PEP 8 issue in exos_facts. Add os_listener module for OpenStack Octavia (#42604) Add os_loadbalancer module for OpenStack Octavia service (#42552) network/exos: add exos_facts module (#43210) Added junos_config note about override set format (#43608) New networking module: voss_command (#43741) make lockfile destination settable and update doc (#42795) VMware: Update example (#43786) Update eos clcionf plugin (#43809) refine aks document (#43810) revert #43546 nxos terminal plugin (#43806) improve 'service' integration tests (#43655) Fix nsg cannot add rule with purge_rules false (#43699) add azure_rm_appserviceplan module (#40906) ...	2018-08-08 18:19:24 -04:00
Adrian Likins	dc91f6a38e	Merge branch 'devel' into mazer_role_loader * devel: (50 commits) Add new module for Redfish APIs (#41656) VMware Module - vmware_guest_move (#42149) Lenovo port to persistence 1 (#43194) VMware: new module: vmware_guest_boot_manager (#40609) fixes #42042 (#42939) VMware: new module: vmware_category_facts (#39894) VMware: Dynamic Inventory plugin (#37456) Validate and reject if csr_path is not supplied when provider is not assertonly (#41385) VMware: new module : vmware_guest_custom_attributes (#38114) VMware: new module: vmware_guest_attribute_defs (#38144) VMware: Fix mark as virtual machine method (#40521) Ironware: Deprecate provider, support network_cli (#43285) feat: Add a enable_accelerated_networking flag in module + tests; fixes #41218 (#42109) fixing aiuth source (#42923) VMware: handle special characters in datacenter name (#42922) VMware: update examples in vmware_vm_shell (#42410) VMWare: refactor vmware_vm_shell module (#39957) VMware: additional VSAN facts about Hostsystem (#40456) nxos cliconf plugin refactor (#43203) Correcting conditionals looping (#43331) ...	2018-07-27 12:40:36 -04:00
Adrian Likins	5d05327449	Support loading mazer/galaxy roles from content path Allow roles to be referenced, found, and loaded by using a galaxy/mazer style role name. ie, "geerlingguy.nginx.nginx' or 'namespace.reponame.contentname' When used with galaxy roles installed via 'mazer' (https://github.com/ansible/mazer) this lets roles to be referenced by the mazer/galaxy style name. See https://galaxy.ansible.com/docs/mazer/examples.html#using-mazer-content for more info on how/where mazer installs galaxy content. mazer installed content lives in ~/.ansible/content/ Inside of ~/.ansible/content, there are directories for each galaxy namespace (more or less the github org or user id used in galaxy roles). For ex ~/.ansible/content/alikins Inside each namespace directory, there will be a directory for each galaxy 'repo' installed. For a traditional galaxy role, this repo dir will have a name that matches the role name. See https://galaxy.ansible.com/docs/mazer/examples.html#installing-roles For example, geerlingguy.apache will get installed to: ~/.ansible/content/geerlingguy/apache For new multi-content style repos (see https://galaxy.ansible.com/docs/mazer/examples.html#installing-multi-role-repositories) the repo level directory name with match the name of the git repo imported to galaxy. For example, for the github repo at https://github.com/atestuseraccount/ansible-testing-content imported to galaxy-qa at https://galaxy-qa.ansible.com/testing/ansible_testing_content, the repo level directory name is 'ansible_testing_content'. Inside the repo level dir, there are directories for each content type supported by galaxy. For example, 'roles'. Inside each content type dir, there will be a dir named for the each instance of that content. For the 'testing' example above, the 'test-role-a' role will be installed to: ~/.ansible/content/testing/ansible_testing_content/roles/test-role-a To use test-role-a in a playbook, it can be referenced as 'testing.ansible_testing_content.test-role-a' For a traditional role (one where the git repo contains only a single role) like geerlingguy.apache, mazer will install it to: ~/.ansible/content/geerlingguy/apache/roles/apache To reference that role in a playbook, there is a full qualified name and a shorter alias. The fully qualified name in this case would be 'geerlingguy.apache.apache'. That is "namespace.reponame.contentname". The shorter alias is 'geerlingguy.apache'. Note that this name format is compatible with using roles installed with ansible-galaxy. ie, 'mynamespace.myrole' will try to look for a role named 'repo' at ~/.ansible/content/mynamespace/myrole/roles/myrole ie, 'geerlingguy.apache' -> ~/.ansible/content/geerlingguy/apache/roles/apache in addition to more specific 'geerlingguy.apache.apache' -> ~/.ansible/content/geerlingguy/apache/roles/apache or 'testing.some_multi_content_repo.some_role' -> ~/.ansible/content/testing/some_multi_content_repo/roles/some_role But NOT: 'testing.some_multi_content_repo' -> role loading error (assuming there isnt also a role named 'some_multi_content_repo') and NOT: 'testing.myrole' -> role loading error (no repo info) - Extract content path role loading to method - Start removing use of installed_role_spec, seems we dont need it. - Eemove use of passed in installed_role_spec ds. We just need the role_name. - Rename local 'ds' to role_name_parts since ds has a meaning elsewhere - Split out the role->content relative path parsing - Add some unit test coverage for playbook.roles.definition	2018-07-25 15:57:13 -04:00
Adrian Likins	0b6e6d3609	Add DEFAULT_CONTENT_PATH config item Based on `21e17e9f6e`	2018-07-25 15:57:13 -04:00
Adrian Likins	62dab4b53d	add some unit tests for general role stuff	2018-07-25 15:57:13 -04:00