This avoids holding open _new_stdin within the parent process, where subsequent WorkerProcess forks will duplicate it, producing significant noise in the FD table of every worker. Fix by overriding start() and moving the work to there, with a finally: to ensure parent FD is closed after start().
This commit is contained in:
parent
c231fc5a7c
commit
81deb8f132
1 changed files with 30 additions and 19 deletions
|
@ -67,25 +67,36 @@ class WorkerProcess(multiprocessing.Process):
|
||||||
self._variable_manager = variable_manager
|
self._variable_manager = variable_manager
|
||||||
self._shared_loader_obj = shared_loader_obj
|
self._shared_loader_obj = shared_loader_obj
|
||||||
|
|
||||||
if sys.stdin.isatty():
|
def _save_stdin(self):
|
||||||
# dupe stdin, if we have one
|
self._new_stdin = os.devnull
|
||||||
self._new_stdin = sys.stdin
|
try:
|
||||||
try:
|
if sys.stdin.isatty() and sys.stdin.fileno() is not None:
|
||||||
fileno = sys.stdin.fileno()
|
try:
|
||||||
if fileno is not None:
|
self._new_stdin = os.fdopen(os.dup(sys.stdin.fileno()))
|
||||||
try:
|
except OSError:
|
||||||
self._new_stdin = os.fdopen(os.dup(fileno))
|
# couldn't dupe stdin, most likely because it's
|
||||||
except OSError:
|
# not a valid file descriptor, so we just rely on
|
||||||
# couldn't dupe stdin, most likely because it's
|
# using the one that was passed in
|
||||||
# not a valid file descriptor, so we just rely on
|
pass
|
||||||
# using the one that was passed in
|
except (AttributeError, ValueError):
|
||||||
pass
|
# couldn't get stdin's fileno, so we just carry on
|
||||||
except (AttributeError, ValueError):
|
pass
|
||||||
# couldn't get stdin's fileno, so we just carry on
|
|
||||||
pass
|
def start(self):
|
||||||
else:
|
'''
|
||||||
# set to /dev/null
|
multiprocessing.Process replaces the worker's stdin with a new file
|
||||||
self._new_stdin = os.devnull
|
opened on os.devnull, but we wish to preserve it if it is connected to
|
||||||
|
a terminal. Therefore dup a copy prior to calling the real start(),
|
||||||
|
ensuring the descriptor is preserved somewhere in the new child, and
|
||||||
|
make sure it is closed in the parent when start() completes.
|
||||||
|
'''
|
||||||
|
|
||||||
|
self._save_stdin()
|
||||||
|
try:
|
||||||
|
return super(WorkerProcess, self).start()
|
||||||
|
finally:
|
||||||
|
if self._new_stdin != os.devnull:
|
||||||
|
self._new_stdin.close()
|
||||||
|
|
||||||
def _hard_exit(self, e):
|
def _hard_exit(self, e):
|
||||||
'''
|
'''
|
||||||
|
|
Loading…
Reference in a new issue