From f76afab6e54a65cfa8db2d8dc330a72f3eb430ad Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Thu, 19 Oct 2017 13:37:22 -0700
Subject: [PATCH] Add new default Docker container for ansible-test. (#31944)
* Add new default Docker container for ansible-test.
* Update ansible-test change classification.
* Update list of disabled pylint rules.
* Fix pylint issues with ansible-test.
---
test/runner/.dockerignore | 3 ++
test/runner/Dockerfile | 39 ++++++++++++++++++++++++
test/runner/docker/deadsnakes.list | 2 ++
test/runner/docker/requirements.sh | 23 ++++++++++++++
test/runner/lib/classification.py | 29 ++++++++++++++++++
test/runner/lib/sanity/__init__.py | 2 +-
test/runner/requirements/constraints.txt | 3 +-
test/runner/requirements/sanity.txt | 2 +-
test/sanity/pylint/disable.txt | 15 +++++++++
9 files changed, 115 insertions(+), 3 deletions(-)
create mode 100644 test/runner/.dockerignore
create mode 100644 test/runner/Dockerfile
create mode 100644 test/runner/docker/deadsnakes.list
create mode 100755 test/runner/docker/requirements.sh
diff --git a/test/runner/.dockerignore b/test/runner/.dockerignore
new file mode 100644
index 00000000000..c1e80151674
--- /dev/null
+++ b/test/runner/.dockerignore
@@ -0,0 +1,3 @@
+*
+!docker
+!requirements
diff --git a/test/runner/Dockerfile b/test/runner/Dockerfile
new file mode 100644
index 00000000000..d08e7f86164
--- /dev/null
+++ b/test/runner/Dockerfile
@@ -0,0 +1,39 @@
+FROM ubuntu:16.04
+
+COPY docker/deadsnakes.list /etc/apt/sources.list.d/deadsnakes.list
+
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776
+
+RUN apt-get update -y && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ gcc \
+ git \
+ libffi-dev \
+ libxml2-dev \
+ libxslt1-dev \
+ locales \
+ make \
+ python2.6-dev \
+ python2.7-dev \
+ python3.5-dev \
+ python3.6-dev \
+ shellcheck \
+ && \
+ apt-get clean
+
+RUN rm /etc/apt/apt.conf.d/docker-clean
+RUN locale-gen en_US.UTF-8
+VOLUME /sys/fs/cgroup /run/lock /run /tmp
+
+ADD https://bootstrap.pypa.io/get-pip.py /tmp/get-pip.py
+
+COPY requirements/*.txt /tmp/requirements/
+COPY docker/requirements.sh /tmp/
+RUN cd /tmp/requirements && /tmp/requirements.sh
+
+RUN ln -s python2.7 /usr/bin/python2
+RUN ln -s python3.6 /usr/bin/python3
+RUN ln -s python3 /usr/bin/python
+
+ENV container=docker
+CMD ["/sbin/init"]
diff --git a/test/runner/docker/deadsnakes.list b/test/runner/docker/deadsnakes.list
new file mode 100644
index 00000000000..32a1fbc14f8
--- /dev/null
+++ b/test/runner/docker/deadsnakes.list
@@ -0,0 +1,2 @@
+deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main
+deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main
diff --git a/test/runner/docker/requirements.sh b/test/runner/docker/requirements.sh
new file mode 100755
index 00000000000..f16e1b7e982
--- /dev/null
+++ b/test/runner/docker/requirements.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+
+python_versions=(
+ 2.6
+ 2.7
+ 3.5
+ 3.6
+)
+
+requirements=()
+
+for requirement in *.txt; do
+ if [ "${requirement}" != "constraints.txt" ]; then
+ requirements+=("-r" "${requirement}")
+ fi
+done
+
+for python_version in "${python_versions[@]}"; do
+ set -x
+ "python${python_version}" /tmp/get-pip.py -c constraints.txt
+ "pip${python_version}" install --disable-pip-version-check -c constraints.txt "${requirements[@]}"
+ set +x
+done
diff --git a/test/runner/lib/classification.py b/test/runner/lib/classification.py
index d63fc1c2901..f7c89bd24ee 100644
--- a/test/runner/lib/classification.py
+++ b/test/runner/lib/classification.py
@@ -461,6 +461,9 @@ class PathMapper(object):
test_path = os.path.dirname(test_path)
+ if path.startswith('test/runner/docker/'):
+ return minimal # not used by tests, only used to build the default container
+
if path.startswith('test/runner/lib/cloud/'):
cloud_target = 'cloud/%s/' % name
@@ -476,6 +479,32 @@ class PathMapper(object):
'sanity': 'all', # test infrastructure, run all sanity checks
}
+ if path.startswith('test/runner/requirements/'):
+ if name in (
+ 'integration',
+ 'network-integration',
+ 'windows-integration',
+ ):
+ return {
+ name: self.integration_all_target,
+ }
+
+ if name in (
+ 'sanity',
+ 'units',
+ ):
+ return {
+ name: 'all',
+ }
+
+ if name.startswith('integration.cloud.'):
+ cloud_target = 'cloud/%s/' % name.split('.')[2]
+
+ if cloud_target in self.integration_targets_by_alias:
+ return {
+ 'integration': cloud_target,
+ }
+
if path.startswith('test/runner/'):
return all_tests(self.args) # test infrastructure, run all tests
diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py
index 0589b1bddd6..0b42aefb251 100644
--- a/test/runner/lib/sanity/__init__.py
+++ b/test/runner/lib/sanity/__init__.py
@@ -83,7 +83,7 @@ def command_sanity(args):
if isinstance(test, SanityMultipleVersion):
versions = SUPPORTED_PYTHON_VERSIONS
else:
- versions = None,
+ versions = (None,)
for version in versions:
if args.python and version and version != args.python:
diff --git a/test/runner/requirements/constraints.txt b/test/runner/requirements/constraints.txt
index b229cc763a1..ef5da0f6a8d 100644
--- a/test/runner/requirements/constraints.txt
+++ b/test/runner/requirements/constraints.txt
@@ -1,6 +1,7 @@
coverage >= 4.2, != 4.3.2 # features in 4.2+ required, avoid known bug in 4.3.2 on python 2.6
pywinrm >= 0.2.1 # 0.1.1 required, but 0.2.1 provides better performance
-pylint >= 1.5.3, < 1.7.0 # 1.4.1 adds JSON output, but 1.5.3 fixes bugs related to JSON output
+pylint == 1.7.4 ; python_version >= '3.5' # versions before 1.7.1 hang or fail to install on python 3.x
+pylint == 1.6.5 ; python_version <= '2.7' # versions after 1.6.5 hang or fail during test on python 2.x
sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later
wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
yamllint != 1.8.0 ; python_version < '2.7' # yamllint 1.8.0 requires python 2.7+ while earlier/later versions do not
diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt
index df8c7c0713c..34d6e07f475 100644
--- a/test/runner/requirements/sanity.txt
+++ b/test/runner/requirements/sanity.txt
@@ -3,7 +3,7 @@ jinja2
mock
paramiko
pycodestyle
-pylint
+pylint ; python_version >= '2.7' # pylint 1.5.3+ is required for non-buggy JSON output, but 1.4+ requires python 2.7+
pytest
rstcheck
sphinx
diff --git a/test/sanity/pylint/disable.txt b/test/sanity/pylint/disable.txt
index c0142960074..2ffed769c82 100644
--- a/test/sanity/pylint/disable.txt
+++ b/test/sanity/pylint/disable.txt
@@ -13,7 +13,9 @@ blacklisted-name
broad-except
cell-var-from-loop
consider-iterating-dictionary
+consider-merging-isinstance
consider-using-enumerate
+consider-using-ternary
deprecated-lambda
deprecated-method
deprecated-module
@@ -29,11 +31,16 @@ global-variable-undefined
import-error
import-self
invalid-name
+invalid-sequence-index
+invalid-unary-operand-type
+len-as-condition
line-too-long
+literal-comparison
locally-disabled
method-hidden
misplaced-comparison-constant
missing-docstring
+no-else-return
no-init
no-member
no-name-in-module
@@ -47,11 +54,14 @@ old-style-class
pointless-statement
pointless-string-statement
protected-access
+pylint
+redefined-argument-from-local
redefined-builtin
redefined-outer-name
redefined-variable-type
reimported
relative-import
+signature-differs
simplifiable-if-statement
super-init-not-called
superfluous-parens
@@ -68,17 +78,22 @@ too-many-nested-blocks
too-many-public-methods
too-many-return-statements
too-many-statements
+trailing-comma-tuple
+unbalanced-tuple-unpacking
undefined-loop-variable
ungrouped-imports
unidiomatic-typecheck
unneeded-not
unsubscriptable-object
+unsupported-assignment-operation
+unsupported-delete-operation
unsupported-membership-test
unused-argument
unused-import
unused-variable
unused-wildcard-import
used-before-assignment
+useless-super-delegation
wildcard-import
wrong-import-order
wrong-import-position