patch ansible-connection collection plugin loading (#59119)

changelogs/fragments/ac_collection_plugin_load.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- fixed collection-based plugin loading in ansible-connection (eg networking plugins)

lib/ansible/executor/task_executor.py

@@ -26,6 +26,7 @@ from ansible.playbook.conditional import Conditional
 from ansible.playbook.task import Task
 from ansible.plugins.loader import become_loader, cliconf_loader, connection_loader, httpapi_loader, netconf_loader, terminal_loader
 from ansible.template import Templar
+from ansible.utils.collection_loader import AnsibleCollectionLoader
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.unsafe_proxy import UnsafeProxy, wrap_var
 from ansible.vars.clean import namespace_facts, clean_facts
@@ -1054,8 +1055,13 @@ def start_connection(play_context, variables):
 
     env = os.environ.copy()
     env.update({
+        # HACK; most of these paths may change during the controller's lifetime
+        # (eg, due to late dynamic role includes, multi-playbook execution), without a way
+        # to invalidate/update, ansible-connection won't always see the same plugins the controller
+        # can.
         'ANSIBLE_BECOME_PLUGINS': become_loader.print_paths(),
         'ANSIBLE_CLICONF_PLUGINS': cliconf_loader.print_paths(),
+        'ANSIBLE_COLLECTIONS_PATHS': os.pathsep.join(AnsibleCollectionLoader().n_collection_paths),
         'ANSIBLE_CONNECTION_PLUGINS': connection_loader.print_paths(),
         'ANSIBLE_HTTPAPI_PLUGINS': httpapi_loader.print_paths(),
         'ANSIBLE_NETCONF_PLUGINS': netconf_loader.print_paths(),

lib/ansible/utils/collection_loader.py

@@ -13,7 +13,8 @@ from types import ModuleType
 
 from ansible import constants as C
 from ansible.module_utils._text import to_bytes, to_native, to_text
-from ansible.module_utils.six import iteritems, string_types
+from ansible.module_utils.six import iteritems, string_types, with_metaclass
+from ansible.utils.singleton import Singleton
 
 # HACK: keep Python 2.6 controller tests happy in CI until they're properly split
 try:
@@ -34,7 +35,7 @@ _collection_qualified_re = re.compile(to_text(r'^(\w+)\.(\w+)\.(\w+)$'))
 
 
 # FIXME: exception handling/error logging
-class AnsibleCollectionLoader(object):
+class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
     def __init__(self):
         self._n_configured_paths = C.config.get_config_value('COLLECTIONS_PATHS')
 
@@ -66,7 +67,7 @@ class AnsibleCollectionLoader(object):
             sys.modules[pkg_name] = newmod
 
     @property
-    def _n_collection_paths(self):
+    def n_collection_paths(self):
         return self._n_playbook_paths + self._n_configured_paths
 
     def set_playbook_paths(self, b_playbook_paths):
@@ -145,7 +146,7 @@ class AnsibleCollectionLoader(object):
                 return newmod
 
         if not parent_pkg:  # top-level package, look for NS subpackages on all collection paths
-            package_paths = [self._extend_path_with_ns(p, fullname) for p in self._n_collection_paths]
+            package_paths = [self._extend_path_with_ns(p, fullname) for p in self.n_collection_paths]
         else:  # subpackage; search in all subpaths (we'll limit later inside a collection)
             package_paths = [self._extend_path_with_ns(p, fullname) for p in parent_pkg.__path__]
 
@@ -309,6 +310,4 @@ def is_collection_ref(candidate_name):
 
 
 def set_collection_playbook_paths(b_playbook_paths):
-    # set for any/all AnsibleCollectionLoader instance(s) on meta_path
+    AnsibleCollectionLoader().set_playbook_paths(b_playbook_paths)
-    for loader in (l for l in sys.meta_path if isinstance(l, AnsibleCollectionLoader)):
-        loader.set_playbook_paths(b_playbook_paths)

test/units/utils/test_collection_loader.py

@@ -23,6 +23,9 @@ def test_import_from_collection(monkeypatch):
 
     from ansible.utils.collection_loader import AnsibleCollectionLoader
 
+    # zap the singleton collection loader instance if it exists
+    AnsibleCollectionLoader._Singleton__instance = None
+
     for index in [idx for idx, obj in enumerate(sys.meta_path) if isinstance(obj, AnsibleCollectionLoader)]:
         # replace any existing collection loaders that may exist
         # since these were loaded during unit test collection