From db182ba49826702d2a8c6824bb1a2a7e5a71afcd Mon Sep 17 00:00:00 2001 From: jeromew Date: Mon, 9 Dec 2013 13:58:28 -0500 Subject: [PATCH 1/2] copy ssh.py to ssh_alt.py --- .../runner/connection_plugins/ssh_alt.py | 311 ++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 lib/ansible/runner/connection_plugins/ssh_alt.py diff --git a/lib/ansible/runner/connection_plugins/ssh_alt.py b/lib/ansible/runner/connection_plugins/ssh_alt.py new file mode 100644 index 00000000000..68c6f17d4a7 --- /dev/null +++ b/lib/ansible/runner/connection_plugins/ssh_alt.py @@ -0,0 +1,311 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# + +import os +import subprocess +import shlex +import pipes +import random +import select +import fcntl +import hmac +import pwd +import gettext +import pty +from hashlib import sha1 +import ansible.constants as C +from ansible.callbacks import vvv +from ansible import errors +from ansible import utils + +class Connection(object): + ''' ssh based connections ''' + + def __init__(self, runner, host, port, user, password, private_key_file, *args, **kwargs): + self.runner = runner + self.host = host + self.ipv6 = ':' in self.host + self.port = port + self.user = user + self.password = password + self.private_key_file = private_key_file + self.HASHED_KEY_MAGIC = "|1|" + + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700) + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) + + def connect(self): + ''' connect to the remote host ''' + + vvv("ESTABLISH CONNECTION FOR USER: %s" % self.user, host=self.host) + + self.common_args = [] + extra_args = C.ANSIBLE_SSH_ARGS + if extra_args is not None: + self.common_args += shlex.split(extra_args) + else: + self.common_args += ["-o", "ControlMaster=auto", + "-o", "ControlPersist=60s", + "-o", "ControlPath=%s" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self.cp_dir))] + + cp_in_use = False + cp_path_set = False + for arg in self.common_args: + if arg.find("ControlPersist") != -1: + cp_in_use = True + if arg.find("ControlPath") != -1: + cp_path_set = True + + if cp_in_use and not cp_path_set: + self.common_args += ["-o", "ControlPath=%s" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self.cp_dir))] + + if not C.HOST_KEY_CHECKING: + self.common_args += ["-o", "StrictHostKeyChecking=no"] + + if self.port is not None: + self.common_args += ["-o", "Port=%d" % (self.port)] + if self.private_key_file is not None: + self.common_args += ["-o", "IdentityFile="+os.path.expanduser(self.private_key_file)] + elif self.runner.private_key_file is not None: + self.common_args += ["-o", "IdentityFile="+os.path.expanduser(self.runner.private_key_file)] + if self.password: + self.common_args += ["-o", "GSSAPIAuthentication=no", + "-o", "PubkeyAuthentication=no"] + else: + self.common_args += ["-o", "KbdInteractiveAuthentication=no", + "-o", "PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey", + "-o", "PasswordAuthentication=no"] + if self.user != pwd.getpwuid(os.geteuid())[0]: + self.common_args += ["-o", "User="+self.user] + self.common_args += ["-o", "ConnectTimeout=%d" % self.runner.timeout] + + return self + + def _password_cmd(self): + if self.password: + try: + p = subprocess.Popen(["sshpass"], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + except OSError: + raise errors.AnsibleError("to use the 'ssh' connection type with passwords, you must install the sshpass program") + (self.rfd, self.wfd) = os.pipe() + return ["sshpass", "-d%d" % self.rfd] + return [] + + def _send_password(self): + if self.password: + os.close(self.rfd) + os.write(self.wfd, "%s\n" % self.password) + os.close(self.wfd) + + def not_in_host_file(self, host): + host_file = os.path.expanduser(os.path.expandvars("~${USER}/.ssh/known_hosts")) + if not os.path.exists(host_file): + print "previous known host file not found" + return True + host_fh = open(host_file) + data = host_fh.read() + host_fh.close() + for line in data.split("\n"): + if line is None or line.find(" ") == -1: + continue + tokens = line.split() + if tokens[0].find(self.HASHED_KEY_MAGIC) == 0: + # this is a hashed known host entry + try: + (kn_salt,kn_host) = tokens[0][len(self.HASHED_KEY_MAGIC):].split("|",2) + hash = hmac.new(kn_salt.decode('base64'), digestmod=sha1) + hash.update(host) + if hash.digest() == kn_host.decode('base64'): + return False + except: + # invalid hashed host key, skip it + continue + else: + # standard host file entry + if host in tokens[0]: + return False + return True + + def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh'): + ''' run a command on the remote host ''' + + ssh_cmd = self._password_cmd() + ssh_cmd += ["ssh", "-tt"] + if utils.VERBOSITY > 3: + ssh_cmd += ["-vvv"] + else: + ssh_cmd += ["-q"] + ssh_cmd += self.common_args + + if self.ipv6: + ssh_cmd += ['-6'] + ssh_cmd += [self.host] + + if not self.runner.sudo or not sudoable: + if executable: + ssh_cmd.append(executable + ' -c ' + pipes.quote(cmd)) + else: + ssh_cmd.append(cmd) + else: + sudocmd, prompt, success_key = utils.make_sudo_cmd(sudo_user, executable, cmd) + ssh_cmd.append(sudocmd) + + vvv("EXEC %s" % ssh_cmd, host=self.host) + + not_in_host_file = self.not_in_host_file(self.host) + + if C.HOST_KEY_CHECKING and not_in_host_file: + # lock around the initial SSH connectivity so the user prompt about whether to add + # the host to known hosts is not intermingled with multiprocess output. + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) + fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) + + + + try: + # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors + master, slave = pty.openpty() + p = subprocess.Popen(ssh_cmd, stdin=slave, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = os.fdopen(master, 'w', 0) + os.close(slave) + except: + p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = p.stdin + + self._send_password() + + if self.runner.sudo and sudoable and self.runner.sudo_pass: + fcntl.fcntl(p.stdout, fcntl.F_SETFL, + fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) + sudo_output = '' + while not sudo_output.endswith(prompt) and success_key not in sudo_output: + rfd, wfd, efd = select.select([p.stdout], [], + [p.stdout], self.runner.timeout) + if p.stdout in rfd: + chunk = p.stdout.read() + if not chunk: + raise errors.AnsibleError('ssh connection closed waiting for sudo password prompt') + sudo_output += chunk + else: + stdout = p.communicate() + raise errors.AnsibleError('ssh connection error waiting for sudo password prompt') + if success_key not in sudo_output: + stdin.write(self.runner.sudo_pass + '\n') + fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) + + # We can't use p.communicate here because the ControlMaster may have stdout open as well + stdout = '' + stderr = '' + rpipes = [p.stdout, p.stderr] + while True: + rfd, wfd, efd = select.select(rpipes, [], rpipes, 1) + + # fail early if the sudo password is wrong + if self.runner.sudo and sudoable and self.runner.sudo_pass: + incorrect_password = gettext.dgettext( + "sudo", "Sorry, try again.") + if stdout.endswith("%s\r\n%s" % (incorrect_password, prompt)): + raise errors.AnsibleError('Incorrect sudo password') + + if p.stdout in rfd: + dat = os.read(p.stdout.fileno(), 9000) + stdout += dat + if dat == '': + rpipes.remove(p.stdout) + if p.stderr in rfd: + dat = os.read(p.stderr.fileno(), 9000) + stderr += dat + if dat == '': + rpipes.remove(p.stderr) + if not rpipes or p.poll() is not None: + p.wait() + break + stdin.close() # close stdin after we read from stdout (see also issue #848) + + if C.HOST_KEY_CHECKING and not_in_host_file: + # lock around the initial SSH connectivity so the user prompt about whether to add + # the host to known hosts is not intermingled with multiprocess output. + fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_UN) + fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_UN) + controlpersisterror = stderr.find('Bad configuration option: ControlPersist') != -1 or stderr.find('unknown configuration option: ControlPersist') != -1 + if p.returncode != 0 and controlpersisterror: + raise errors.AnsibleError('using -c ssh on certain older ssh versions may not support ControlPersist, set ANSIBLE_SSH_ARGS="" (or ansible_ssh_args in the config file) before running again') + + return (p.returncode, '', stdout, stderr) + + def put_file(self, in_path, out_path): + ''' transfer a file from local to remote ''' + vvv("PUT %s TO %s" % (in_path, out_path), host=self.host) + if not os.path.exists(in_path): + raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path) + cmd = self._password_cmd() + + host = self.host + if self.ipv6: + host = '[%s]' % host + + if C.DEFAULT_SCP_IF_SSH: + cmd += ["scp"] + self.common_args + cmd += [in_path,host + ":" + pipes.quote(out_path)] + indata = None + else: + cmd += ["sftp"] + self.common_args + [host] + indata = "put %s %s\n" % (pipes.quote(in_path), pipes.quote(out_path)) + + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._send_password() + stdout, stderr = p.communicate(indata) + + if p.returncode != 0: + raise errors.AnsibleError("failed to transfer file to %s:\n%s\n%s" % (out_path, stdout, stderr)) + + def fetch_file(self, in_path, out_path): + ''' fetch a file from remote to local ''' + vvv("FETCH %s TO %s" % (in_path, out_path), host=self.host) + cmd = self._password_cmd() + + host = self.host + if self.ipv6: + host = '[%s]' % host + + if C.DEFAULT_SCP_IF_SSH: + cmd += ["scp"] + self.common_args + cmd += [host + ":" + in_path, out_path] + indata = None + else: + cmd += ["sftp"] + self.common_args + [host] + indata = "get %s %s\n" % (in_path, out_path) + + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._send_password() + stdout, stderr = p.communicate(indata) + + if p.returncode != 0: + raise errors.AnsibleError("failed to transfer file from %s:\n%s\n%s" % (in_path, stdout, stderr)) + + def close(self): + ''' not applicable since we're executing openssh binaries ''' + pass + From 7f8863f96db53144c73dfb79edce5173fa362b35 Mon Sep 17 00:00:00 2001 From: jeromew Date: Mon, 9 Dec 2013 13:59:41 -0500 Subject: [PATCH 2/2] ssh_alt.py / decrease # of ssh roundtrips --- lib/ansible/runner/__init__.py | 91 +++++++++++++++---- lib/ansible/runner/action_plugins/add_host.py | 2 +- lib/ansible/runner/action_plugins/assemble.py | 2 + lib/ansible/runner/action_plugins/copy.py | 2 + lib/ansible/runner/action_plugins/debug.py | 2 +- lib/ansible/runner/action_plugins/fail.py | 2 +- lib/ansible/runner/action_plugins/group_by.py | 2 +- .../runner/action_plugins/include_vars.py | 2 +- lib/ansible/runner/action_plugins/raw.py | 2 +- lib/ansible/runner/action_plugins/script.py | 2 + lib/ansible/runner/action_plugins/set_fact.py | 2 +- lib/ansible/runner/action_plugins/template.py | 2 + .../runner/action_plugins/unarchive.py | 2 + .../runner/connection_plugins/accelerate.py | 6 +- .../runner/connection_plugins/chroot.py | 6 +- .../runner/connection_plugins/fireball.py | 6 +- .../runner/connection_plugins/funcd.py | 6 +- lib/ansible/runner/connection_plugins/jail.py | 6 +- .../runner/connection_plugins/local.py | 6 +- .../runner/connection_plugins/paramiko_ssh.py | 6 +- lib/ansible/runner/connection_plugins/ssh.py | 6 +- .../runner/connection_plugins/ssh_alt.py | 46 +++++++--- 22 files changed, 160 insertions(+), 49 deletions(-) diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index df9c3cacf1f..6c5f52b4f69 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -286,7 +286,7 @@ class Runner(object): def _execute_module(self, conn, tmp, module_name, args, async_jid=None, async_module=None, async_limit=None, inject=None, persist_files=False, complex_args=None): - ''' runs a module that has already been transferred ''' + ''' transfer and run a module along with its arguments on the remote side''' # hack to support fireball mode if module_name == 'fireball': @@ -294,12 +294,24 @@ class Runner(object): if 'port' not in args: args += " port=%s" % C.ZEROMQ_PORT - (remote_module_path, module_style, shebang) = self._copy_module(conn, tmp, module_name, args, inject, complex_args) + ( + module_style, + shebang, + module_data + ) = self._configure_module(conn, module_name, args, inject, complex_args) + + # a remote tmp path may be necessary and not already created + if self._late_needs_tmp_path(conn, tmp, module_style): + tmp = self._make_tmp_path(conn) + + remote_module_path = os.path.join(tmp, module_name) + + if not conn.has_pipelining: + self._transfer_str(conn, tmp, module_name, module_data) environment_string = self._compute_environment_string(inject) - cmd_mod = "" - if self.sudo and self.sudo_user != 'root': + if tmp.find("tmp") != -1 and self.sudo and self.sudo_user != 'root': # deal with possible umask issues once sudo'ed to other user cmd_chmod = "chmod a+r %s" % remote_module_path self._low_level_exec_command(conn, cmd_chmod, tmp, sudoable=False) @@ -343,6 +355,9 @@ class Runner(object): if not shebang: raise errors.AnsibleError("module is missing interpreter line") + if conn.has_pipelining: + cmd = "" + cmd = " ".join([environment_string.strip(), shebang.replace("#!","").strip(), cmd]) cmd = cmd.strip() @@ -357,12 +372,16 @@ class Runner(object): # specified in the play, not the sudo_user sudoable = False - res = self._low_level_exec_command(conn, cmd, tmp, sudoable=sudoable) + in_data = None + if conn.has_pipelining: + in_data = module_data - if self.sudo and self.sudo_user != 'root': + res = self._low_level_exec_command(conn, cmd, tmp, sudoable=sudoable, in_data=in_data) + + if tmp.find("tmp") != -1 and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files: + if self.sudo and self.sudo_user != 'root': # not sudoing to root, so maybe can't delete files as that other user # have to clean up temp files as original user in a second step - if tmp.find("tmp") != -1 and not C.DEFAULT_KEEP_REMOTE_FILES and not persist_files: cmd2 = "rm -rf %s >/dev/null 2>&1" % tmp self._low_level_exec_command(conn, cmd2, tmp, sudoable=False) @@ -670,8 +689,8 @@ class Runner(object): return ReturnData(host=host, comm_ok=False, result=result) tmp = '' - # all modules get a tempdir, action plugins get one unless they have NEEDS_TMPPATH set to False - if getattr(handler, 'NEEDS_TMPPATH', True): + # action plugins may DECLARE via TRANSFERS_FILES = True that they need a remote tmp path working dir + if self._early_needs_tmp_path(module_name, handler): tmp = self._make_tmp_path(conn) # render module_args and complex_args templates @@ -697,7 +716,7 @@ class Runner(object): delay = float(delay) time.sleep(delay) tmp = '' - if getattr(handler, 'NEEDS_TMPPATH', True): + if self._early_needs_tmp_path(module_name, handler): tmp = self._make_tmp_path(conn) result = handler.run(conn, tmp, module_name, module_args, inject, complex_args) result.result['attempts'] = x @@ -753,9 +772,29 @@ class Runner(object): self.callbacks.on_ok(host, data) return result + def _early_needs_tmp_path(self, module_name, handler): + ''' detect if a tmp path should be created before the handler is called ''' + if module_name in utils.plugins.action_loader: + return getattr(handler, 'TRANSFERS_FILES', False) + # other modules never need tmp path at early stage + return False + + def _late_needs_tmp_path(self, conn, tmp, module_style): + if tmp.find("tmp") != -1: + # tmp has already been created + return False + if not conn.has_pipelining: + # tmp is necessary to store the module source code + return True + if module_style != "new": + # even when conn has pipelining, old style modules need tmp to store arguments + return True + return False + + # ***************************************************** - def _low_level_exec_command(self, conn, cmd, tmp, sudoable=False, executable=None): + def _low_level_exec_command(self, conn, cmd, tmp, sudoable=False, executable=None, in_data=None): ''' execute a command string over SSH, return the output ''' if executable is None: @@ -768,7 +807,7 @@ class Runner(object): if conn.user == sudo_user: sudoable = False - rc, stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudo_user, sudoable=sudoable, executable=executable) + rc, stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudo_user, sudoable=sudoable, executable=executable, in_data=in_data) if type(stdout) not in [ str, unicode ]: out = ''.join(stdout.readlines()) @@ -858,27 +897,39 @@ class Runner(object): raise errors.AnsibleError('failed to resolve remote temporary directory from %s: `%s` returned empty string' % (basetmp, cmd)) return rc - # ***************************************************** def _copy_module(self, conn, tmp, module_name, module_args, inject, complex_args=None): ''' transfer a module over SFTP, does not run it ''' + ( + module_style, + module_shebang, + module_data + ) = self._configure_module(conn, module_name, module_args, inject, complex_args) + module_remote_path = os.path.join(tmp, module_name) + + self._transfer_str(conn, tmp, module_name, module_data) + + return (module_remote_path, module_style, module_shebang) + + # ***************************************************** + + def _configure_module(self, conn, module_name, module_args, inject, complex_args=None): + ''' find module and configure it ''' # Search module path(s) for named module. - in_path = utils.plugins.module_finder.find_plugin(module_name) - if in_path is None: + module_path = utils.plugins.module_finder.find_plugin(module_name) + if module_path is None: raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths())) - out_path = os.path.join(tmp, module_name) # insert shared code and arguments into the module - (module_data, module_style, shebang) = module_replacer.modify_module( - in_path, complex_args, module_args, inject + (module_data, module_style, module_shebang) = module_replacer.modify_module( + module_path, complex_args, module_args, inject ) - self._transfer_str(conn, tmp, module_name, module_data) + return (module_style, module_shebang, module_data) - return (out_path, module_style, shebang) # ***************************************************** diff --git a/lib/ansible/runner/action_plugins/add_host.py b/lib/ansible/runner/action_plugins/add_host.py index ef27d99275d..bcd1a90ae98 100644 --- a/lib/ansible/runner/action_plugins/add_host.py +++ b/lib/ansible/runner/action_plugins/add_host.py @@ -29,7 +29,7 @@ class ActionModule(object): ### We need to be able to modify the inventory BYPASS_HOST_LOOP = True - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/assemble.py b/lib/ansible/runner/action_plugins/assemble.py index ad6caf3c6f8..3f55023aa9c 100644 --- a/lib/ansible/runner/action_plugins/assemble.py +++ b/lib/ansible/runner/action_plugins/assemble.py @@ -26,6 +26,8 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): + TRANSFERS_FILES = True + def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/copy.py b/lib/ansible/runner/action_plugins/copy.py index 00f12a8bc1c..e65cab44268 100644 --- a/lib/ansible/runner/action_plugins/copy.py +++ b/lib/ansible/runner/action_plugins/copy.py @@ -35,6 +35,8 @@ sys.setdefaultencoding("utf8") class ActionModule(object): + TRANSFERS_FILES = True + def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/debug.py b/lib/ansible/runner/action_plugins/debug.py index ee9c6ec32bb..93879c1a8cd 100644 --- a/lib/ansible/runner/action_plugins/debug.py +++ b/lib/ansible/runner/action_plugins/debug.py @@ -24,7 +24,7 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): ''' Print statements during execution ''' - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/fail.py b/lib/ansible/runner/action_plugins/fail.py index 7c3a58cfb91..2bbaf40313c 100644 --- a/lib/ansible/runner/action_plugins/fail.py +++ b/lib/ansible/runner/action_plugins/fail.py @@ -23,7 +23,7 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): ''' Fail with custom message ''' - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/group_by.py b/lib/ansible/runner/action_plugins/group_by.py index 2385df3863f..f8b4f318db2 100644 --- a/lib/ansible/runner/action_plugins/group_by.py +++ b/lib/ansible/runner/action_plugins/group_by.py @@ -28,7 +28,7 @@ class ActionModule(object): ### We need to be able to modify the inventory BYPASS_HOST_LOOP = True - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/include_vars.py b/lib/ansible/runner/action_plugins/include_vars.py index 5a1f69a6c50..344cc82fa62 100644 --- a/lib/ansible/runner/action_plugins/include_vars.py +++ b/lib/ansible/runner/action_plugins/include_vars.py @@ -23,7 +23,7 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/raw.py b/lib/ansible/runner/action_plugins/raw.py index 7fcfe92927a..ac6ed698d54 100644 --- a/lib/ansible/runner/action_plugins/raw.py +++ b/lib/ansible/runner/action_plugins/raw.py @@ -23,7 +23,7 @@ from ansible import errors from ansible.runner.return_data import ReturnData class ActionModule(object): - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/script.py b/lib/ansible/runner/action_plugins/script.py index c65e8cc8d18..6b584c15b63 100644 --- a/lib/ansible/runner/action_plugins/script.py +++ b/lib/ansible/runner/action_plugins/script.py @@ -26,6 +26,8 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): + TRANSFERS_FILES = True + def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/set_fact.py b/lib/ansible/runner/action_plugins/set_fact.py index 95d8cd6ef5c..37502ce3a2f 100644 --- a/lib/ansible/runner/action_plugins/set_fact.py +++ b/lib/ansible/runner/action_plugins/set_fact.py @@ -20,7 +20,7 @@ from ansible.runner.return_data import ReturnData class ActionModule(object): - NEEDS_TMPPATH = False + TRANSFERS_FILES = False def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/template.py b/lib/ansible/runner/action_plugins/template.py index 9ddc3340a46..b34c14ec6a5 100644 --- a/lib/ansible/runner/action_plugins/template.py +++ b/lib/ansible/runner/action_plugins/template.py @@ -25,6 +25,8 @@ import base64 class ActionModule(object): + TRANSFERS_FILES = True + def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/action_plugins/unarchive.py b/lib/ansible/runner/action_plugins/unarchive.py index f0b9dee945c..9733469ba63 100644 --- a/lib/ansible/runner/action_plugins/unarchive.py +++ b/lib/ansible/runner/action_plugins/unarchive.py @@ -35,6 +35,8 @@ import pipes class ActionModule(object): + TRANSFERS_FILES = True + def __init__(self, runner): self.runner = runner diff --git a/lib/ansible/runner/connection_plugins/accelerate.py b/lib/ansible/runner/connection_plugins/accelerate.py index 085c1662880..d260a634379 100644 --- a/lib/ansible/runner/connection_plugins/accelerate.py +++ b/lib/ansible/runner/connection_plugins/accelerate.py @@ -49,6 +49,7 @@ class Connection(object): self.port = port[0] self.accport = port[1] self.is_connected = False + self.has_pipelining = False if not self.port: self.port = constants.DEFAULT_REMOTE_PORT @@ -158,9 +159,12 @@ class Connection(object): except socket.timeout: raise errors.AnsibleError("timed out while waiting to receive data") - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + if executable == "": executable = constants.DEFAULT_EXECUTABLE diff --git a/lib/ansible/runner/connection_plugins/chroot.py b/lib/ansible/runner/connection_plugins/chroot.py index 305e9eb06b5..1080ea54b57 100644 --- a/lib/ansible/runner/connection_plugins/chroot.py +++ b/lib/ansible/runner/connection_plugins/chroot.py @@ -30,6 +30,7 @@ class Connection(object): def __init__(self, runner, host, port, *args, **kwargs): self.chroot = host + self.has_pipelining = False if os.geteuid() != 0: raise errors.AnsibleError("chroot connection requires running as root") @@ -59,9 +60,12 @@ class Connection(object): return self - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the chroot ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + # We enter chroot as root so sudo stuff can be ignored if executable: diff --git a/lib/ansible/runner/connection_plugins/fireball.py b/lib/ansible/runner/connection_plugins/fireball.py index 727d6d279f7..40848edead4 100644 --- a/lib/ansible/runner/connection_plugins/fireball.py +++ b/lib/ansible/runner/connection_plugins/fireball.py @@ -37,6 +37,7 @@ class Connection(object): def __init__(self, runner, host, port, *args, **kwargs): self.runner = runner + self.has_pipelining = False # attempt to work around shared-memory funness if getattr(self.runner, 'aes_keys', None): @@ -67,9 +68,12 @@ class Connection(object): return self - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + vvv("EXEC COMMAND %s" % cmd) if self.runner.sudo and sudoable: diff --git a/lib/ansible/runner/connection_plugins/funcd.py b/lib/ansible/runner/connection_plugins/funcd.py index 5acdb4a30cf..a5dc631ef89 100644 --- a/lib/ansible/runner/connection_plugins/funcd.py +++ b/lib/ansible/runner/connection_plugins/funcd.py @@ -42,6 +42,7 @@ class Connection(object): def __init__(self, runner, host, port, *args, **kwargs): self.runner = runner self.host = host + self.has_pipelining = False # port is unused, this go on func self.port = port @@ -53,9 +54,12 @@ class Connection(object): return self def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, - executable='/bin/sh'): + executable='/bin/sh', in_data=None): ''' run a command on the remote minion ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + vvv("EXEC %s" % (cmd), host=self.host) p = self.client.command.run(cmd)[self.host] return (p[0], '', p[1], p[2]) diff --git a/lib/ansible/runner/connection_plugins/jail.py b/lib/ansible/runner/connection_plugins/jail.py index b750874f06a..89b13bbc445 100644 --- a/lib/ansible/runner/connection_plugins/jail.py +++ b/lib/ansible/runner/connection_plugins/jail.py @@ -60,6 +60,7 @@ class Connection(object): self.jail = host self.runner = runner self.host = host + self.has_pipelining = False if os.geteuid() != 0: raise errors.AnsibleError("jail connection requires running as root") @@ -90,9 +91,12 @@ class Connection(object): local_cmd = '%s "%s" %s' % (self.jexec_cmd, self.jail, cmd) return local_cmd - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the chroot ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + # We enter chroot as root so sudo stuff can be ignored local_cmd = self._generate_cmd(executable, cmd) diff --git a/lib/ansible/runner/connection_plugins/local.py b/lib/ansible/runner/connection_plugins/local.py index d62f2e4c06c..0cf7da42be6 100644 --- a/lib/ansible/runner/connection_plugins/local.py +++ b/lib/ansible/runner/connection_plugins/local.py @@ -34,15 +34,19 @@ class Connection(object): self.host = host # port is unused, since this is local self.port = port + self.has_pipelining = False def connect(self, port=None): ''' connect to the local host; nothing to do here ''' return self - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the local host ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + if not self.runner.sudo or not sudoable: if executable: local_cmd = [executable, '-c', cmd] diff --git a/lib/ansible/runner/connection_plugins/paramiko_ssh.py b/lib/ansible/runner/connection_plugins/paramiko_ssh.py index abf9dafc8f7..667e03c0e88 100644 --- a/lib/ansible/runner/connection_plugins/paramiko_ssh.py +++ b/lib/ansible/runner/connection_plugins/paramiko_ssh.py @@ -121,6 +121,7 @@ class Connection(object): self.user = user self.password = password self.private_key_file = private_key_file + self.has_pipelining = False def _cache_key(self): return "%s__%s__" % (self.host, self.user) @@ -175,9 +176,12 @@ class Connection(object): return ssh - def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user, sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + bufsize = 4096 try: chan = self.ssh.get_transport().open_session() diff --git a/lib/ansible/runner/connection_plugins/ssh.py b/lib/ansible/runner/connection_plugins/ssh.py index 68c6f17d4a7..b630a76aea6 100644 --- a/lib/ansible/runner/connection_plugins/ssh.py +++ b/lib/ansible/runner/connection_plugins/ssh.py @@ -45,6 +45,7 @@ class Connection(object): self.password = password self.private_key_file = private_key_file self.HASHED_KEY_MAGIC = "|1|" + self.has_pipelining = False fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700) @@ -144,9 +145,12 @@ class Connection(object): return False return True - def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' + if in_data: + raise errors.AnsibleError("Internal Error: this module does not support optimized module pipelining") + ssh_cmd = self._password_cmd() ssh_cmd += ["ssh", "-tt"] if utils.VERBOSITY > 3: diff --git a/lib/ansible/runner/connection_plugins/ssh_alt.py b/lib/ansible/runner/connection_plugins/ssh_alt.py index 68c6f17d4a7..96a81f922e0 100644 --- a/lib/ansible/runner/connection_plugins/ssh_alt.py +++ b/lib/ansible/runner/connection_plugins/ssh_alt.py @@ -45,6 +45,7 @@ class Connection(object): self.password = password self.private_key_file = private_key_file self.HASHED_KEY_MAGIC = "|1|" + self.has_pipelining = True fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) self.cp_dir = utils.prepare_writeable_dir('$HOME/.ansible/cp',mode=0700) @@ -144,11 +145,13 @@ class Connection(object): return False return True - def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh'): + def exec_command(self, cmd, tmp_path, sudo_user,sudoable=False, executable='/bin/sh', in_data=None): ''' run a command on the remote host ''' ssh_cmd = self._password_cmd() - ssh_cmd += ["ssh", "-tt"] + ssh_cmd += ["ssh", "-C"] + if not in_data: + ssh_cmd += ["-tt"] if utils.VERBOSITY > 3: ssh_cmd += ["-vvv"] else: @@ -178,19 +181,26 @@ class Connection(object): fcntl.lockf(self.runner.process_lockfile, fcntl.LOCK_EX) fcntl.lockf(self.runner.output_lockfile, fcntl.LOCK_EX) - - - try: - # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors - master, slave = pty.openpty() - p = subprocess.Popen(ssh_cmd, stdin=slave, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdin = os.fdopen(master, 'w', 0) - os.close(slave) - except: + # create process + if in_data: + # do not use pseudo-pty p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdin = p.stdin + else: + # try to use upseudo-pty + try: + # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors + master, slave = pty.openpty() + p = subprocess.Popen(ssh_cmd, stdin=slave, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = os.fdopen(master, 'w', 0) + os.close(slave) + except: + p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdin = p.stdin + self._send_password() @@ -198,6 +208,9 @@ class Connection(object): fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) | os.O_NONBLOCK) sudo_output = '' + if in_data: + # no terminal => no prompt on output. process is waiting for sudo_pass + stdin.write(self.runner.sudo_pass + '\n') while not sudo_output.endswith(prompt) and success_key not in sudo_output: rfd, wfd, efd = select.select([p.stdout], [], [p.stdout], self.runner.timeout) @@ -212,11 +225,16 @@ class Connection(object): if success_key not in sudo_output: stdin.write(self.runner.sudo_pass + '\n') fcntl.fcntl(p.stdout, fcntl.F_SETFL, fcntl.fcntl(p.stdout, fcntl.F_GETFL) & ~os.O_NONBLOCK) - # We can't use p.communicate here because the ControlMaster may have stdout open as well stdout = '' stderr = '' rpipes = [p.stdout, p.stderr] + if in_data: + try: + stdin.write(in_data) + stdin.close() + except: + raise errors.AnsibleError('SSH Error: data could not be sent to the remote host. Make sure this host can be reached over ssh') while True: rfd, wfd, efd = select.select(rpipes, [], rpipes, 1)