This replaces the previous method for capping poorly Popen() performance, instead entirely monkey-patching the problem function rather than simply working around it.
This commit is contained in:
parent
f203744b48
commit
87ddead0b2
|
@ -43,7 +43,6 @@ import operator
|
||||||
import os
|
import os
|
||||||
import pwd
|
import pwd
|
||||||
import re
|
import re
|
||||||
import resource
|
|
||||||
import signal
|
import signal
|
||||||
import stat
|
import stat
|
||||||
import subprocess
|
import subprocess
|
||||||
|
@ -91,15 +90,41 @@ _fork_parent = None
|
||||||
good_temp_dir = None
|
good_temp_dir = None
|
||||||
|
|
||||||
|
|
||||||
# issue #362: subprocess.Popen(close_fds=True) aka. AnsibleModule.run_command()
|
def subprocess__Popen__close_fds(self, but):
|
||||||
# loops the entire SC_OPEN_MAX space. CentOS>5 ships with 1,048,576 FDs by
|
"""
|
||||||
# default, resulting in huge (>500ms) runtime waste running many commands.
|
issue #362, #435: subprocess.Popen(close_fds=True) aka.
|
||||||
# Therefore if we are a child, cap the range to something reasonable.
|
AnsibleModule.run_command() loops the entire FD space on Python<3.2.
|
||||||
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
|
CentOS>5 ships with 1,048,576 FDs by default, resulting in huge (>500ms)
|
||||||
if (rlimit[0] > 4096 or rlimit[1] > 4096) and not mitogen.is_master:
|
latency starting children. Therefore replace Popen._close_fds on Linux with
|
||||||
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, 4096))
|
a version that is O(fds) rather than O(_SC_OPEN_MAX).
|
||||||
subprocess.MAXFD = 4096 # Python <3.x
|
"""
|
||||||
del rlimit
|
try:
|
||||||
|
names = os.listdir('/proc/self/fd')
|
||||||
|
except OSError:
|
||||||
|
# May fail if acting on a container that does not have /proc mounted.
|
||||||
|
self._original_close_fds(but)
|
||||||
|
return
|
||||||
|
|
||||||
|
for name in names:
|
||||||
|
if not name.isdigit():
|
||||||
|
continue
|
||||||
|
|
||||||
|
fd = int(name, 10)
|
||||||
|
if fd > 2 and fd != but:
|
||||||
|
try:
|
||||||
|
os.close(fd)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if (
|
||||||
|
sys.platform.startswith('linux') and
|
||||||
|
sys.version < '3.0' and
|
||||||
|
hasattr(subprocess.Popen, '_close_fds') and
|
||||||
|
not mitogen.is_master
|
||||||
|
):
|
||||||
|
subprocess.Popen._original_close_fds = subprocess.Popen._close_fds
|
||||||
|
subprocess.Popen._close_fds = subprocess__Popen__close_fds
|
||||||
|
|
||||||
|
|
||||||
def get_small_file(context, path):
|
def get_small_file(context, path):
|
||||||
|
|
Loading…
Reference in New Issue