From 27fc049993b8c26792dd7e5fa96e46da281a71d7 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 27 Mar 2020 12:24:08 -0700 Subject: [PATCH] Fix ansible-test code coverage collection. (#68502) The last task in a play should now properly report code coverage. This change should also eliminate empty coverage files, as well as incomplete coverage files resulting from early worker termination. --- .../ansible-test-coverage-incomplete.yml | 3 +++ lib/ansible/config/base.yml | 19 +++++++++++++++++++ lib/ansible/executor/task_queue_manager.py | 10 ++++++++++ .../ansible_test/_internal/ansible_util.py | 5 +++++ 4 files changed, 37 insertions(+) create mode 100644 changelogs/fragments/ansible-test-coverage-incomplete.yml diff --git a/changelogs/fragments/ansible-test-coverage-incomplete.yml b/changelogs/fragments/ansible-test-coverage-incomplete.yml new file mode 100644 index 00000000000..1327366d8b4 --- /dev/null +++ b/changelogs/fragments/ansible-test-coverage-incomplete.yml @@ -0,0 +1,3 @@ +bugfixes: + - ansible-test now correctly collects code coverage on the last task in a play. + This should resolve issues with missing code coverage, empty coverage files and corrupted coverage files resulting from early worker termination. diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index dca9009d6dc..e2c5aaa7294 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1828,6 +1828,25 @@ TAGS_SKIP: ini: - {key: skip, section: tags} version_added: "2.5" +WORKER_SHUTDOWN_POLL_COUNT: + name: Worker Shutdown Poll Count + default: 0 + description: + - The maximum number of times to check Task Queue Manager worker processes to verify they have exited cleanly. + - After this limit is reached any worker processes still running will be terminated. + - This is for internal use only. + env: [{name: ANSIBLE_WORKER_SHUTDOWN_POLL_COUNT}] + type: integer + version_added: '2.10' +WORKER_SHUTDOWN_POLL_DELAY: + name: Worker Shutdown Poll Delay + default: 0.1 + description: + - The number of seconds to sleep between polling loops when checking Task Queue Manager worker processes to verify they have exited cleanly. + - This is for internal use only. + env: [{name: ANSIBLE_WORKER_SHUTDOWN_POLL_DELAY}] + type: float + version_added: '2.10' USE_PERSISTENT_CONNECTIONS: name: Persistence default: False diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index fabc4a92370..30c5f383a67 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -21,6 +21,7 @@ __metaclass__ = type import os import tempfile +import time from ansible import constants as C from ansible import context @@ -256,6 +257,15 @@ class TaskQueueManager: def _cleanup_processes(self): if hasattr(self, '_workers'): + for attempts_remaining in range(C.WORKER_SHUTDOWN_POLL_COUNT - 1, -1, -1): + if not any(worker_prc and worker_prc.is_alive() for worker_prc in self._workers): + break + + if attempts_remaining: + time.sleep(C.WORKER_SHUTDOWN_POLL_DELAY) + else: + display.warning('One or more worker processes are still running and will be terminated.') + for worker_prc in self._workers: if worker_prc and worker_prc.is_alive(): try: diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index b57cb1cfc6a..22eef704cd1 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -77,6 +77,11 @@ def ansible_environment(args, color=True, ansible_config=None): PYTHONPATH=get_ansible_python_path(), PAGER='/bin/cat', PATH=path, + # give TQM worker processes time to report code coverage results + # without this the last task in a play may write no coverage file, an empty file, or an incomplete file + # enabled even when not using code coverage to surface warnings when worker processes do not exit cleanly + ANSIBLE_WORKER_SHUTDOWN_POLL_COUNT='100', + ANSIBLE_WORKER_SHUTDOWN_POLL_DELAY='0.1', ) if isinstance(args, IntegrationConfig) and args.coverage: