[stable-2.10] Fix ansible-test handling of egg-info. (#73595)

* Add test to verify pkg_resources imports work.

(cherry picked from commit 133a29acb4)

* [stable-2.10] Fix ansible-test handling of egg-info.

Resolves https://github.com/ansible/ansible/issues/67990.
(cherry picked from commit d092356fc5)

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/ansible-test-egg-info-handling.yml

@@ -0,0 +1,11 @@
+bugfixes:
+  - ansible-test - Running tests using an installed version of ``ansible-test`` against one Python version from another no longer fails
+                   due to a missing ``egg-info`` directory.
+                   This could occur when testing plugins which import ``pkg_resources``.
+  - ansible-test - Running tests using an installed version of ``ansible-test`` no longer generates an error attempting to create an ``egg-info`` directory
+                   when an existing one is not found in the expected location.
+                   This could occur if the existing ``egg-info`` directory included a Python version specifier in the name.
+minor_changes:
+  - ansible-test - Generation of an ``egg-info`` directory, if needed, is now done after installing test dependencies and before running tests.
+                   When running from an installed version of ``ansible-test`` a temporary directory is used to avoid permissions issues.
+                   Previously it was done before installing test dependencies and adjacent to the installed directory.

test/integration/targets/pkg_resources/aliases

@@ -0,0 +1 @@
+shippable/posix/group1

test/integration/targets/pkg_resources/lookup_plugins/check_pkg_resources.py

@@ -0,0 +1,23 @@
+"""
+This test case verifies that pkg_resources imports from ansible plugins are functional.
+
+If pkg_resources is not installed this test will succeed.
+If pkg_resources is installed but is unable to function, this test will fail.
+
+One known failure case this test can detect is when ansible declares a __requires__ and then tests are run without an egg-info directory.
+"""
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+# noinspection PyUnresolvedReferences
+try:
+    from pkg_resources import Requirement
+except ImportError:
+    Requirement = None
+
+from ansible.plugins.lookup import LookupBase
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        return []

test/integration/targets/pkg_resources/tasks/main.yml

@@ -0,0 +1,3 @@
+- name: Verify that pkg_resources imports are functional
+  debug:
+    msg: "{{ lookup('check_pkg_resources') }}"

test/lib/ansible_test/_internal/ansible_util.py

@@ -11,6 +11,10 @@ from .constants import (
     SOFT_RLIMIT_NOFILE,
 )
 
+from .io import (
+    write_text_file,
+)
+
 from .util import (
     common_environment,
     display,
@@ -20,6 +24,7 @@ from .util import (
     ANSIBLE_TEST_DATA_ROOT,
     ANSIBLE_BIN_PATH,
     ANSIBLE_SOURCE_ROOT,
+    get_ansible_version,
 )
 
 from .util_common import (
@@ -74,7 +79,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_CONFIG=ansible_config,
         ANSIBLE_LIBRARY='/dev/null',
         ANSIBLE_DEVEL_WARNING='false',  # Don't show warnings that CI is running devel
-        PYTHONPATH=get_ansible_python_path(),
+        PYTHONPATH=get_ansible_python_path(args),
         PAGER='/bin/cat',
         PATH=path,
         # give TQM worker processes time to report code coverage results
@@ -170,28 +175,59 @@ def configure_plugin_paths(args):  # type: (CommonConfig) -> t.Dict[str, str]
     return env
 
 
-def get_ansible_python_path():  # type: () -> str
+def get_ansible_python_path(args):  # type: (CommonConfig) -> str
     """
     Return a directory usable for PYTHONPATH, containing only the ansible package.
     If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit.
     """
-    if ANSIBLE_SOURCE_ROOT:
-        # when running from source there is no need for a temporary directory to isolate the ansible package
-        return os.path.dirname(ANSIBLE_LIB_ROOT)
-
     try:
         return get_ansible_python_path.python_path
     except AttributeError:
         pass
 
-    python_path = create_temp_dir(prefix='ansible-test-')
+    if ANSIBLE_SOURCE_ROOT:
+        # when running from source there is no need for a temporary directory to isolate the ansible package
+        python_path = os.path.dirname(ANSIBLE_LIB_ROOT)
+    else:
+        # when not running from source the installed directory is unsafe to add to PYTHONPATH
+        # doing so would expose many unwanted packages on sys.path
+        # instead a temporary directory is created which contains only ansible using a symlink
+        python_path = create_temp_dir(prefix='ansible-test-')
+
+        os.symlink(ANSIBLE_LIB_ROOT, os.path.join(python_path, 'ansible'))
+
+    if not args.explain:
+        generate_egg_info(python_path)
+
     get_ansible_python_path.python_path = python_path
 
-    os.symlink(ANSIBLE_LIB_ROOT, os.path.join(python_path, 'ansible'))
-
     return python_path
 
 
+def generate_egg_info(path):  # type: (str) -> None
+    """Generate an egg-info in the specified base directory."""
+    # minimal PKG-INFO stub following the format defined in PEP 241
+    # required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography
+    # newer setuptools versions are happy with an empty directory
+    # including a stub here means we don't need to locate the existing file or have setup.py generate it when running from source
+    pkg_info = '''
+Metadata-Version: 1.0
+Name: ansible
+Version: %s
+Platform: UNKNOWN
+Summary: Radically simple IT automation
+Author-email: info@ansible.com
+License: GPLv3+
+''' % get_ansible_version()
+
+    pkg_info_path = os.path.join(path, 'ansible_base.egg-info', 'PKG-INFO')
+
+    if os.path.exists(pkg_info_path):
+        return
+
+    write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True)
+
+
 def check_pyyaml(args, version, required=True, quiet=False):
     """
     :type args: EnvironmentConfig

test/lib/ansible_test/_internal/executor.py

@@ -278,8 +278,6 @@ def install_command_requirements(args, python_version=None, context=None, enable
         if args.raw:
             return
 
-    generate_egg_info(args)
-
     if not args.requirements:
         return
 
@@ -425,46 +423,6 @@ def pip_list(args, pip):
     return stdout
 
 
-def generate_egg_info(args):
-    """
-    :type args: EnvironmentConfig
-    """
-    if args.explain:
-        return
-
-    ansible_version = get_ansible_version()
-
-    # inclusion of the version number in the path is optional
-    # see: https://setuptools.readthedocs.io/en/latest/formats.html#filename-embedded-metadata
-    egg_info_path = ANSIBLE_LIB_ROOT + '_base-%s.egg-info' % ansible_version
-
-    if os.path.exists(egg_info_path):
-        return
-
-    egg_info_path = ANSIBLE_LIB_ROOT + '_base.egg-info'
-
-    if os.path.exists(egg_info_path):
-        return
-
-    # minimal PKG-INFO stub following the format defined in PEP 241
-    # required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography
-    # newer setuptools versions are happy with an empty directory
-    # including a stub here means we don't need to locate the existing file or have setup.py generate it when running from source
-    pkg_info = '''
-Metadata-Version: 1.0
-Name: ansible
-Version: %s
-Platform: UNKNOWN
-Summary: Radically simple IT automation
-Author-email: info@ansible.com
-License: GPLv3+
-''' % get_ansible_version()
-
-    pkg_info_path = os.path.join(egg_info_path, 'PKG-INFO')
-
-    write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True)
-
-
 def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True, context=None):
     """
     :type pip: list[str]