From de63cba7e88ea509930f8f695fcdc2b2fe61be2a Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 15 Jun 2020 16:22:25 -0700 Subject: [PATCH] transparent downstream vendoring (#69850) * builtin downstream vendoring support * allows downstream packagers to install packages to `ansible/_vendor` that will automatically be added to head of sys.path during `ansible` package load * tests * sort conflicting package names in warning text * sanity fixes * skip unnecessary comparison --- changelogs/fragments/downstream_vendoring.yml | 2 + lib/ansible/__init__.py | 3 + lib/ansible/_vendor/__init__.py | 46 +++++++++++++ test/units/_vendor/test_vendor.py | 65 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 changelogs/fragments/downstream_vendoring.yml create mode 100644 lib/ansible/_vendor/__init__.py create mode 100644 test/units/_vendor/test_vendor.py diff --git a/changelogs/fragments/downstream_vendoring.yml b/changelogs/fragments/downstream_vendoring.yml new file mode 100644 index 00000000000..e3a41f87d14 --- /dev/null +++ b/changelogs/fragments/downstream_vendoring.yml @@ -0,0 +1,2 @@ +minor_changes: +- downstream packagers may install packages under ansible._vendor, which will be added to head of sys.path at ansible package load diff --git a/lib/ansible/__init__.py b/lib/ansible/__init__.py index d8aeac5e583..e4905a18532 100644 --- a/lib/ansible/__init__.py +++ b/lib/ansible/__init__.py @@ -19,6 +19,9 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +# make vendored top-level modules accessible EARLY +import ansible._vendor + # Note: Do not add any code to this file. The ansible module may be # a namespace package when using Ansible-2.1+ Anything in this file may not be # available if one of the other packages in the namespace is loaded first. diff --git a/lib/ansible/_vendor/__init__.py b/lib/ansible/_vendor/__init__.py new file mode 100644 index 00000000000..e6a4c56d6ab --- /dev/null +++ b/lib/ansible/_vendor/__init__.py @@ -0,0 +1,46 @@ +# (c) 2020 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pkgutil +import sys +import warnings + +# This package exists to host vendored top-level Python packages for downstream packaging. Any Python packages +# installed beneath this one will be masked from the Ansible loader, and available from the front of sys.path. +# It is expected that the vendored packages will be loaded very early, so a warning will be fired on import of +# the top-level ansible package if any packages beneath this are already loaded at that point. +# +# Python packages may be installed here during downstream packaging using something like: +# pip install --upgrade -t (path to this dir) cryptography pyyaml packaging jinja2 + +# mask vendored content below this package from being accessed as an ansible subpackage +__path__ = [] + + +def _ensure_vendored_path_entry(): + """ + Ensure that any downstream-bundled content beneath this package is available at the top of sys.path + """ + # patch our vendored dir onto sys.path + vendored_path_entry = os.path.dirname(__file__) + vendored_module_names = set(m[1] for m in pkgutil.iter_modules([vendored_path_entry], '')) # m[1] == m.name + + if vendored_module_names: + # patch us early to load vendored deps transparently + if vendored_path_entry in sys.path: + # handle reload case by removing the existing entry, wherever it might be + sys.path.remove(vendored_path_entry) + sys.path.insert(0, vendored_path_entry) + + already_loaded_vendored_modules = set(sys.modules.keys()).intersection(vendored_module_names) + + if already_loaded_vendored_modules: + warnings.warn('One or more Python packages bundled by this ansible-base distribution were already ' + 'loaded ({0}). This may result in undefined behavior.'.format(', '.join(sorted(already_loaded_vendored_modules)))) + + +_ensure_vendored_path_entry() diff --git a/test/units/_vendor/test_vendor.py b/test/units/_vendor/test_vendor.py new file mode 100644 index 00000000000..6a0fa38551d --- /dev/null +++ b/test/units/_vendor/test_vendor.py @@ -0,0 +1,65 @@ +# (c) 2020 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import pkgutil +import pytest +import sys + +from units.compat.mock import MagicMock, NonCallableMagicMock, patch + + +def reset_internal_vendor_package(): + import ansible + ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor') + + if ansible_vendor_path in sys.path: + sys.path.remove(ansible_vendor_path) + + for pkg in ['ansible._vendor', 'ansible']: + if pkg in sys.modules: + del sys.modules[pkg] + + +def test_package_path_masking(): + from ansible import _vendor + + assert hasattr(_vendor, '__path__') and _vendor.__path__ == [] + + +def test_no_vendored(): + reset_internal_vendor_package() + with patch.object(pkgutil, 'iter_modules', return_value=[]): + previous_path = list(sys.path) + import ansible + ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor') + + assert ansible_vendor_path not in sys.path + assert sys.path == previous_path + + +def test_vendored(vendored_pkg_names=None): + if not vendored_pkg_names: + vendored_pkg_names = ['boguspkg'] + reset_internal_vendor_package() + with patch.object(pkgutil, 'iter_modules', return_value=list((None, p, None) for p in vendored_pkg_names)): + previous_path = list(sys.path) + import ansible + ansible_vendor_path = os.path.join(os.path.dirname(ansible.__file__), '_vendor') + assert sys.path[0] == ansible_vendor_path + + if ansible_vendor_path in previous_path: + previous_path.remove(ansible_vendor_path) + + assert sys.path[1:] == previous_path + + +def test_vendored_conflict(): + with pytest.warns(UserWarning) as w: + import pkgutil + import sys + test_vendored(vendored_pkg_names=['sys', 'pkgutil']) # pass a real package we know is already loaded + assert 'pkgutil, sys' in str(w[0].message) # ensure both conflicting modules are listed and sorted