From bbef250c2bde5e8c4aeda8629a74ddc3fa55614e Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Thu, 12 Nov 2020 14:39:08 -0500 Subject: [PATCH] Fix super annoying Python 2.6 multiprocessing.Queue stack trace in CI (#72604) * Fix super annoying Python 2.6 multiprocessing.Queue stack trace in CI A bug exists in Python 2.6 that sometimes raises an exception during interpreter shutdown. We encounter this frequently in our CI since we run tests on CentOS 6 as the control node, which has Python 2.6.6 with this bug. This PR adds a very minor sleep only on Python 2.6 which gets around this issue. I did lot of testing using a standalon script I found that easily duplicated the issue to find the minimum sleep value needed to avoid this issue. CPython issue: https://bugs.python.org/issue4106 Fix in CPython: https://hg.python.org/cpython/rev/d316315a8781 * Use correct attribute --- .../fragments/py26-multiprocess-queue-bug.yml | 2 ++ lib/ansible/executor/task_queue_manager.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 changelogs/fragments/py26-multiprocess-queue-bug.yml diff --git a/changelogs/fragments/py26-multiprocess-queue-bug.yml b/changelogs/fragments/py26-multiprocess-queue-bug.yml new file mode 100644 index 00000000000..c61cc55564d --- /dev/null +++ b/changelogs/fragments/py26-multiprocess-queue-bug.yml @@ -0,0 +1,2 @@ +bugfixes: + - account for bug in Python 2.6 that occurs during interpreter shutdown to avoid stack trace diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index 9b4f51537ad..5647e4e6da5 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -20,6 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os +import sys import tempfile import threading import time @@ -328,6 +329,22 @@ class TaskQueueManager: self._final_q.close() self._cleanup_processes() + # A bug exists in Python 2.6 that causes an exception to be raised during + # interpreter shutdown. This is only an issue in our CI testing but we + # hit it frequently enough to add a small sleep to avoid the issue. + # This can be removed once we have split controller available in CI. + # + # Further information: + # Issue: https://bugs.python.org/issue4106 + # Fix: https://hg.python.org/cpython/rev/d316315a8781 + # + try: + if (2, 6) == (sys.version_info[0:2]): + time.sleep(0.0001) + except (IndexError, AttributeError): + # In case there is an issue getting the version info, don't raise an Exception + pass + def _cleanup_processes(self): if hasattr(self, '_workers'): for attempts_remaining in range(C.WORKER_SHUTDOWN_POLL_COUNT - 1, -1, -1):