cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

ssh_alt.py / decrease # of ssh roundtrips

Browse source
This commit is contained in:
jeromew 2013-12-11 04:55:56 -05:00
parent dc41912158
commit 5c965a75f0
23 changed files with 175 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
lib/ansible/runner/__init__.py
View file

@ -286,7 +286,7 @@ class Runner(object):
def _execute_module(self, conn, tmp, module_name, args,
async_jid=None, async_module=None, async_limit=None, inject=None, persist_files=False, complex_args=None):
''' runs a module that has already been transferred '''
''' transfer and run a module along with its arguments on the remote side'''
# hack to support fireball mode
if module_name == 'fireball':
@ -294,17 +294,32 @@ class Runner(object):
if 'port' not in args:
args += " port=%s" % C.ZEROMQ_PORT
(remote_module_path, module_style, shebang) = self._copy_module(conn, tmp, module_name, args, inject, complex_args)
(
module_style,
shebang,
module_data
) = self._configure_module(conn, module_name, args, inject, complex_args)
# a remote tmp path may be necessary and not already created
if self._late_needs_tmp_path(conn, tmp, module_style):
tmp = self._make_tmp_path(conn)
remote_module_path = os.path.join(tmp, module_name)
if (module_style != 'new'
or async_jid is not None
or not conn.has_pipelining):
self._transfer_str(conn, tmp, module_name, module_data)
environment_string = self._compute_environment_string(inject)
cmd_mod = ""
if self.sudo and self.sudo_user != 'root':
if tmp.find("tmp") != -1 and self.sudo and self.sudo_user != 'root':
# deal with possible umask issues once sudo'ed to other user
cmd_chmod = "chmod a+r %s" % remote_module_path
self._low_level_exec_command(conn, cmd_chmod, tmp, sudoable=False)
cmd = ""
in_data = None
if module_style != 'new':
if 'CHECKMODE=True' in args:
# if module isn't using AnsibleModuleCommon infrastructure we can't be certain it knows how to
@ -336,13 +351,17 @@ class Runner(object):
cmd = " ".join([str(x) for x in [remote_module_path, async_jid, async_limit, async_module, argsfile]])
else:
if async_jid is None:
cmd = "%s" % (remote_module_path)
if conn.has_pipelining:
in_data = module_data
else:
cmd = "%s" % (remote_module_path)
else:
cmd = " ".join([str(x) for x in [remote_module_path, async_jid, async_limit, async_module]])
if not shebang:
raise errors.AnsibleError("module is missing interpreter line")
cmd = " ".join([environment_string.strip(), shebang.replace("#!","").strip(), cmd])
cmd = cmd.strip()
@ -357,12 +376,12 @@ class Runner(object):
# specified in the play, not the sudo_user
sudoable = False
res = self._low_level_exec_command(conn, cmd, tmp, sudoable=sudoable)
res = self._low_level_exec_command(conn, cmd, tmp, sudoable=sudoable, in_data=in_data)
if self.sudo and self.sudo_user != 'root':
if tmp.find("tmp") != -1 and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files:
if self.sudo and self.sudo_user != 'root':
# not sudoing to root, so maybe can't delete files as that other user
# have to clean up temp files as original user in a second step
if tmp.find("tmp") != -1 and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files:
cmd2 = "rm -rf %s >/dev/null 2>&1" % tmp
self._low_level_exec_command(conn, cmd2, tmp, sudoable=False)
@ -415,7 +434,7 @@ class Runner(object):
host_variables = self.inventory.get_variables(host)
host_connection = host_variables.get('ansible_connection', self.transport)
if host_connection in [ 'paramiko', 'ssh', 'accelerate' ]:
if host_connection in [ 'paramiko', 'ssh', 'ssh_alt', 'accelerate' ]:
port = host_variables.get('ansible_ssh_port', self.remote_port)
if port is None:
port = C.DEFAULT_REMOTE_PORT
@ -602,7 +621,7 @@ class Runner(object):
if not self.accelerate_port:
self.accelerate_port = C.ACCELERATE_PORT
if actual_transport in [ 'paramiko', 'ssh', 'accelerate' ]:
if actual_transport in [ 'paramiko', 'ssh', 'ssh_alt', 'accelerate' ]:
actual_port = inject.get('ansible_ssh_port', port)
# the delegated host may have different SSH port configured, etc
@ -670,8 +689,8 @@ class Runner(object):
return ReturnData(host=host, comm_ok=False, result=result)
tmp = ''
# all modules get a tempdir, action plugins get one unless they have NEEDS_TMPPATH set to False
if getattr(handler, 'NEEDS_TMPPATH', True):
# action plugins may DECLARE via TRANSFERS_FILES = True that they need a remote tmp path working dir
if self._early_needs_tmp_path(module_name, handler):
tmp = self._make_tmp_path(conn)
# render module_args and complex_args templates
@ -697,7 +716,7 @@ class Runner(object):
delay = float(delay)
time.sleep(delay)
tmp = ''
if getattr(handler, 'NEEDS_TMPPATH', True):
if self._early_needs_tmp_path(module_name, handler):
tmp = self._make_tmp_path(conn)
result = handler.run(conn, tmp, module_name, module_args, inject, complex_args)
result.result['attempts'] = x
@ -753,9 +772,29 @@ class Runner(object):
self.callbacks.on_ok(host, data)
return result
def _early_needs_tmp_path(self, module_name, handler):
''' detect if a tmp path should be created before the handler is called '''
if module_name in utils.plugins.action_loader:
return getattr(handler, 'TRANSFERS_FILES', False)
# other modules never need tmp path at early stage
return False
def _late_needs_tmp_path(self, conn, tmp, module_style):
if tmp.find("tmp") != -1:
# tmp has already been created
return False
if not conn.has_pipelining:
# tmp is necessary to store the module source code
return True
if module_style != "new":
# even when conn has pipelining, old style modules need tmp to store arguments
return True
return False
# *****************************************************
def _low_level_exec_command(self, conn, cmd, tmp, sudoable=False, executable=None):
def _low_level_exec_command(self, conn, cmd, tmp, sudoable=False, executable=None, in_data=None):
''' execute a command string over SSH, return the output '''
if executable is None:
@ -768,7 +807,7 @@ class Runner(object):
if conn.user == sudo_user:
sudoable = False
rc, stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudo_user, sudoable=sudoable, executable=executable)
rc, stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudo_user, sudoable=sudoable, executable=executable, in_data=in_data)
if type(stdout) not in [ str, unicode ]:
out = ''.join(stdout.readlines())
@ -840,7 +879,7 @@ class Runner(object):
if result['rc'] != 0:
if result['rc'] == 5:
output = 'Authentication failure.'
elif result['rc'] == 255 and self.transport == 'ssh':
elif result['rc'] == 255 and self.transport in ['ssh', 'ssh_alt']:
if utils.VERBOSITY > 3:
output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr'])
else:
@ -858,27 +897,39 @@ class Runner(object):
raise errors.AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basetmp, cmd))
return rc
# *****************************************************
def _copy_module(self, conn, tmp, module_name, module_args, inject, complex_args=None):
''' transfer a module over SFTP, does not run it '''
# Search module path(s) for named module.
in_path = utils.plugins.module_finder.find_plugin(module_name)
if in_path is None:
raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths()))
out_path = os.path.join(tmp, module_name)
# insert shared code and arguments into the module
(module_data, module_style, shebang) = module_replacer.modify_module(
in_path, complex_args, module_args, inject
)
(
module_style,
module_shebang,
module_data
) = self._configure_module(conn, module_name, module_args, inject, complex_args)
module_remote_path = os.path.join(tmp, module_name)
self._transfer_str(conn, tmp, module_name, module_data)
return (out_path, module_style, shebang)
return (module_remote_path, module_style, module_shebang)
# *****************************************************
def _configure_module(self, conn, module_name, module_args, inject, complex_args=None):
''' find module and configure it '''
# Search module path(s) for named module.
module_path = utils.plugins.module_finder.find_plugin(module_name)
if module_path is None:
raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths()))
# insert shared code and arguments into the module
(module_data, module_style, module_shebang) = module_replacer.modify_module(
module_path, complex_args, module_args, inject
)
return (module_style, module_shebang, module_data)
# *****************************************************

2
lib/ansible/runner/action_plugins/add_host.py
View file

@ -29,7 +29,7 @@ class ActionModule(object):
### We need to be able to modify the inventory
BYPASS_HOST_LOOP = True
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/assemble.py
View file

@ -26,6 +26,8 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
TRANSFERS_FILES = True
def __init__(self, runner):
self.runner = runner

5
lib/ansible/runner/action_plugins/async.py
View file

@ -19,8 +19,6 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
TRANSFER_FILES = True
def __init__(self, runner):
self.runner = runner
@ -35,6 +33,9 @@ class ActionModule(object):
module_name = 'command'
module_args += " #USE_SHELL"
if tmp.find("tmp") == -1:
tmp = self.runner._make_tmp_path(conn)
(module_path, is_new_style, shebang) = self.runner._copy_module(conn, tmp, module_name, module_args, inject, complex_args=complex_args)
self.runner._low_level_exec_command(conn, "chmod a+rx %s" % module_path, tmp)

2
lib/ansible/runner/action_plugins/copy.py
View file

@ -35,6 +35,8 @@ sys.setdefaultencoding("utf8")
class ActionModule(object):
TRANSFERS_FILES = True
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/debug.py
View file

@ -24,7 +24,7 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
''' Print statements during execution '''
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/fail.py
View file

@ -23,7 +23,7 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
''' Fail with custom message '''
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/group_by.py
View file

@ -28,7 +28,7 @@ class ActionModule(object):
### We need to be able to modify the inventory
BYPASS_HOST_LOOP = True
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/include_vars.py
View file

@ -23,7 +23,7 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/raw.py
View file

@ -23,7 +23,7 @@ from ansible import errors
from ansible.runner.return_data import ReturnData
class ActionModule(object):
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/script.py
View file

@ -26,6 +26,8 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
TRANSFERS_FILES = True
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/set_fact.py
View file

@ -20,7 +20,7 @@ from ansible.runner.return_data import ReturnData
class ActionModule(object):
NEEDS_TMPPATH = False
TRANSFERS_FILES = False
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/template.py
View file

@ -25,6 +25,8 @@ import base64
class ActionModule(object):
TRANSFERS_FILES = True
def __init__(self, runner):
self.runner = runner

2
lib/ansible/runner/action_plugins/unarchive.py
View file

@ -35,6 +35,8 @@ import pipes
class ActionModule(object):
TRANSFERS_FILES = True
def __init__(self, runner):
self.runner = runner

6
lib/ansible/runner/connection_plugins/accelerate.py
View file

@ -49,6 +49,7 @@ class Connection(object):
self.port = port[0]
self.accport = port[1]
self.is_connected = False
self.has_pipelining = False
if not self.port:
self.port = constants.DEFAULT_REMOTE_PORT
@ -158,9 +159,12 @@ class Connection(object):
except socket.timeout:
raise errors.AnsibleError("timed out while waiting to receive data")
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the remote host '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
if executable == "":
executable = constants.DEFAULT_EXECUTABLE

6
lib/ansible/runner/connection_plugins/chroot.py
View file

@ -30,6 +30,7 @@ class Connection(object):
def __init__(self, runner, host, port, *args, **kwargs):
self.chroot = host
self.has_pipelining = False
if os.geteuid() != 0:
raise errors.AnsibleError("chroot connection requires running as root")
@ -59,9 +60,12 @@ class Connection(object):
return self
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the chroot '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
# We enter chroot as root so sudo stuff can be ignored
if executable:

6
lib/ansible/runner/connection_plugins/fireball.py
View file

@ -37,6 +37,7 @@ class Connection(object):
def __init__(self, runner, host, port, *args, **kwargs):
self.runner = runner
self.has_pipelining = False
# attempt to work around shared-memory funness
if getattr(self.runner, 'aes_keys', None):
@ -67,9 +68,12 @@ class Connection(object):
return self
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the remote host '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
vvv("EXEC COMMAND %s" % cmd)
if self.runner.sudo and sudoable:

6
lib/ansible/runner/connection_plugins/funcd.py
View file

@ -42,6 +42,7 @@ class Connection(object):
def __init__(self, runner, host, port, *args, **kwargs):
self.runner = runner
self.host = host
self.has_pipelining = False
# port is unused, this go on func
self.port = port
@ -53,9 +54,12 @@ class Connection(object):
return self
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False,
executable='/bin/sh'):
executable='/bin/sh', in_data=None):
''' run a command on the remote minion '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
vvv("EXEC %s" % (cmd), host=self.host)
p = self.client.command.run(cmd)[self.host]
return (p[0], '', p[1], p[2])

6
lib/ansible/runner/connection_plugins/jail.py
View file

@ -60,6 +60,7 @@ class Connection(object):
self.jail = host
self.runner = runner
self.host = host
self.has_pipelining = False
if os.geteuid() != 0:
raise errors.AnsibleError("jail connection requires running as root")
@ -90,9 +91,12 @@ class Connection(object):
local_cmd = '%s "%s" %s' % (self.jexec_cmd, self.jail, cmd)
return local_cmd
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the chroot '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
# We enter chroot as root so sudo stuff can be ignored
local_cmd = self._generate_cmd(executable, cmd)

6
lib/ansible/runner/connection_plugins/local.py
View file

@ -34,15 +34,19 @@ class Connection(object):
self.host = host
# port is unused, since this is local
self.port = port
self.has_pipelining = False
def connect(self, port=None):
''' connect to the local host; nothing to do here '''
return self
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the local host '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
if not self.runner.sudo or not sudoable:
if executable:
local_cmd = [executable, '-c', cmd]

6
lib/ansible/runner/connection_plugins/paramiko_ssh.py
View file

@ -121,6 +121,7 @@ class Connection(object):
self.user = user
self.password = password
self.private_key_file = private_key_file
self.has_pipelining = False
def _cache_key(self):
return "%s__%s__" % (self.host, self.user)
@ -175,9 +176,12 @@ class Connection(object):
return ssh
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the remote host '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
bufsize = 4096
try:
chan = self.ssh.get_transport().open_session()

6
lib/ansible/runner/connection_plugins/ssh.py
View file

@ -45,6 +45,7 @@ class Connection(object):
self.password = password
self.private_key_file = private_key_file
self.HASHED_KEY_MAGIC = "|1|"
self.has_pipelining = False
fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700)
@ -144,9 +145,12 @@ class Connection(object):
return False
return True
def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the remote host '''
if in_data:
raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining")
ssh_cmd = self._password_cmd()
ssh_cmd += ["ssh", "-tt"]
if utils.VERBOSITY > 3:

56
lib/ansible/runner/connection_plugins/ssh_alt.py
View file

@ -45,6 +45,7 @@ class Connection(object):
self.password = password
self.private_key_file = private_key_file
self.HASHED_KEY_MAGIC = "|1|"
self.has_pipelining = True
fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700)
@ -144,11 +145,13 @@ class Connection(object):
return False
return True
def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh'):
def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh', in_data=None):
''' run a command on the remote host '''
ssh_cmd = self._password_cmd()
ssh_cmd += ["ssh", "-tt"]
ssh_cmd += ["ssh", "-C"]
if not in_data:
ssh_cmd += ["-tt"]
if utils.VERBOSITY > 3:
ssh_cmd += ["-vvv"]
else:
@ -178,19 +181,26 @@ class Connection(object):
fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX)
fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX)
try:
# Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors
master, slave = pty.openpty()
p = subprocess.Popen(ssh_cmd, stdin=slave,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = os.fdopen(master, 'w', 0)
os.close(slave)
except:
# create process
if in_data:
# do not use pseudo-pty
p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = p.stdin
else:
# try to use upseudo-pty
try:
# Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors
master, slave = pty.openpty()
p = subprocess.Popen(ssh_cmd, stdin=slave,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = os.fdopen(master, 'w', 0)
os.close(slave)
except:
p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdin = p.stdin
self._send_password()
@ -198,6 +208,9 @@ class Connection(object):
fcntl.fcntl(p.stdout, fcntl.F_SETFL,
fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
sudo_output = ''
if in_data:
# no terminal => no prompt on output. process is waiting for sudo_pass
stdin.write(self.runner.sudo_pass + '\n')
while not sudo_output.endswith(prompt) and success_key not in sudo_output:
rfd, wfd, efd = select.select([p.stdout], [],
[p.stdout], self.runner.timeout)
@ -212,11 +225,16 @@ class Connection(object):
if success_key not in sudo_output:
stdin.write(self.runner.sudo_pass + '\n')
fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK)
# We can't use p.communicate here because the ControlMaster may have stdout open as well
stdout = ''
stderr = ''
rpipes = [p.stdout, p.stderr]
if in_data:
try:
stdin.write(in_data)
stdin.close()
except:
raise errors.AnsibleError('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')
while True:
rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)
@ -237,9 +255,13 @@ class Connection(object):
stderr += dat
if dat == '':
rpipes.remove(p.stderr)
if not rpipes or p.poll() is not None:
p.wait()
# only break out if we've emptied the pipes, or there is nothing to
# read from and the process has finished.
if (not rpipes or not rfd) and p.poll() is not None:
break
# Calling wait while there are still pipes to read can cause a lock
elif not rpipes and p.poll() == None:
p.wait()
stdin.close() # close stdin after we read from stdout (see also issue #848)
if C.HOST_KEY_CHECKING and not_in_host_file:
@ -250,6 +272,8 @@ class Connection(object):
controlpersisterror = stderr.find('Bad configuration option: ControlPersist') != -1 or stderr.find('unknown configuration option: ControlPersist') != -1
if p.returncode != 0 and controlpersisterror:
raise errors.AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ansible_ssh_args in the config file) before running again')
if p.returncode != 0 and in_data:
raise errors.AnsibleError('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh')
return (p.returncode, '', stdout, stderr)