Avoid WorkerProcess._new_stdin FD sharing (#51623) (#51624)

This avoids holding open _new_stdin within the parent process, where
subsequent WorkerProcess forks will duplicate it, producing significant
noise in the FD table of every worker.

Fix by overriding start() and moving the work to there, with a finally:
to ensure parent FD is closed after start().
This commit is contained in:
dw 2019-03-26 14:41:45 +00:00 committed by James Cammarata
parent c231fc5a7c
commit 81deb8f132

View file

@ -67,25 +67,36 @@ class WorkerProcess(multiprocessing.Process):
self._variable_manager = variable_manager self._variable_manager = variable_manager
self._shared_loader_obj = shared_loader_obj self._shared_loader_obj = shared_loader_obj
if sys.stdin.isatty(): def _save_stdin(self):
# dupe stdin, if we have one self._new_stdin = os.devnull
self._new_stdin = sys.stdin try:
try: if sys.stdin.isatty() and sys.stdin.fileno() is not None:
fileno = sys.stdin.fileno() try:
if fileno is not None: self._new_stdin = os.fdopen(os.dup(sys.stdin.fileno()))
try: except OSError:
self._new_stdin = os.fdopen(os.dup(fileno)) # couldn't dupe stdin, most likely because it's
except OSError: # not a valid file descriptor, so we just rely on
# couldn't dupe stdin, most likely because it's # using the one that was passed in
# not a valid file descriptor, so we just rely on pass
# using the one that was passed in except (AttributeError, ValueError):
pass # couldn't get stdin's fileno, so we just carry on
except (AttributeError, ValueError): pass
# couldn't get stdin's fileno, so we just carry on
pass def start(self):
else: '''
# set to /dev/null multiprocessing.Process replaces the worker's stdin with a new file
self._new_stdin = os.devnull opened on os.devnull, but we wish to preserve it if it is connected to
a terminal. Therefore dup a copy prior to calling the real start(),
ensuring the descriptor is preserved somewhere in the new child, and
make sure it is closed in the parent when start() completes.
'''
self._save_stdin()
try:
return super(WorkerProcess, self).start()
finally:
if self._new_stdin != os.devnull:
self._new_stdin.close()
def _hard_exit(self, e): def _hard_exit(self, e):
''' '''