CI and compat fixes for Jinja2 >= 3.0 (#74666)

* Add constraint for MarkupSafe

MarkupSafe >= 2.0.0 requires Python >= 3.6.0. Add a constraint for older Python versions
and fix the `groupby_filter` test.

* Fix template_jinja2_latest test.

* patch filter decorators on newer Jinja2

* Jinja2 >= 3.0 renames several filter decorators used by Ansible itself, as well as by filters in collections. This patch ensures that the old names are usable within Ansible and by collections without warnings or errors.

* Ignore docs-build issues.

Co-authored-by: Matt Clay <matt@mystile.com>
Co-authored-by: Matt Davis <mrd@redhat.com>

changelogs/fragments/ansible-test-markupsafe-constraint.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible_test - add constraint for ``MarkupSafe`` (https://github.com/ansible/ansible/pull/74666)

changelogs/fragments/jinja2_decorator_renames.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - filter plugins - patch new versions of Jinja2 to prevent warnings/errors on renamed filter decorators (https://github.com/ansible/ansible/issues/74667)

lib/ansible/__init__.py

@@ -22,6 +22,17 @@ __metaclass__ = type
 # make vendored top-level modules accessible EARLY
 import ansible._vendor
 
+# patch Jinja2 >= 3.0 for backwards compatibility
+try:
+    import sys as _sys
+    from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx
+    _mod = _sys.modules['jinja2.filters']
+    _mod.contextfilter = _passctx
+    _mod.environmentfilter = _passenv
+    _mod.evalcontextfilter = _passevalctx
+except ImportError:
+    _sys = None
+
 # Note: Do not add any code to this file.  The ansible module may be
 # a namespace package when using Ansible-2.1+ Anything in this file may not be
 # available if one of the other packages in the namespace is loaded first.

test/integration/targets/groupby_filter/aliases

@@ -1 +1,2 @@
 shippable/posix/group2
+needs/file/test/lib/ansible_test/_data/requirements/constraints.txt

test/integration/targets/groupby_filter/requirements.txt

@@ -0,0 +1,4 @@
+# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
+# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
+pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
+pip >= 7.1 ; python_version >= '2.7' # sanity_ok

test/integration/targets/groupby_filter/runme.sh

@@ -4,10 +4,13 @@ set -eux
 
 source virtualenv.sh
 
-pip install -U jinja2==2.9.4
+# Update pip in the venv to a version that supports constraints
+pip install --requirement requirements.txt
+
+pip install -U jinja2==2.9.4 --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
 
-pip install -U "jinja2<2.9.0"
+pip install -U "jinja2<2.9.0" --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"

test/integration/targets/template_jinja2_latest/aliases

@@ -2,3 +2,4 @@ needs/root
 shippable/posix/group2
 needs/target/template
 skip/aix
+needs/file/test/lib/ansible_test/_data/requirements/constraints.txt

test/integration/targets/template_jinja2_latest/pip-requirements.txt

@@ -0,0 +1,4 @@
+# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
+# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
+pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
+pip >= 7.1 ; python_version >= '2.7' # sanity_ok

test/integration/targets/template_jinja2_latest/runme.sh

@@ -4,7 +4,9 @@ set -eux
 
 source virtualenv.sh
 
-pip install -U -r requirements.txt
+pip install --requirement pip-requirements.txt
+
+pip install -U -r requirements.txt --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
 
 ANSIBLE_ROLES_PATH=../
 export ANSIBLE_ROLES_PATH

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -47,3 +47,4 @@ setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require pyth
 setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
 gssapi < 1.6.0 ; python_version <= '2.7' # gssapi 1.6.0 and later require python 3 or later
 pyspnego >= 0.1.6 ; python_version >= '3.10' # bug in older releases breaks on Python 3.10
+MarkupSafe < 2.0.0 ; python_version < '3.6' # MarkupSafe >= 2.0.0. requires Python >= 3.6

test/sanity/ignore.txt

@@ -1,4 +1,6 @@
 docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes
+docs/docsite/rst/user_guide/playbooks_filters.rst docs-build
+docs/docsite/rst/user_guide/playbooks_python_version.rst docs-build
 examples/play.yml shebang
 examples/scripts/ConfigureRemotingForAnsible.ps1 pslint:PSCustomUseLiteralPath
 examples/scripts/my_test.py shebang # example module but not in a normal module location