Add pipeline-ish method using dd for file transfer over SSH (#18642)
This commit is contained in:
parent
2a90963833
commit
ac51266e8f
5 changed files with 58 additions and 19 deletions
|
@ -331,12 +331,20 @@
|
||||||
#
|
#
|
||||||
#pipelining = False
|
#pipelining = False
|
||||||
|
|
||||||
# Control the mechanism for transferring files
|
# Control the mechanism for transferring files (old)
|
||||||
# * smart = try sftp and then try scp [default]
|
# * smart = try sftp and then try scp [default]
|
||||||
# * True = use scp only
|
# * True = use scp only
|
||||||
# * False = use sftp only
|
# * False = use sftp only
|
||||||
#scp_if_ssh = smart
|
#scp_if_ssh = smart
|
||||||
|
|
||||||
|
# Control the mechanism for transferring files (new)
|
||||||
|
# If set, this will override the scp_if_ssh option
|
||||||
|
# * sftp = use sftp to transfer files
|
||||||
|
# * scp = use scp to transfer files
|
||||||
|
# * piped = use 'dd' over SSH to transfer files
|
||||||
|
# * smart = try sftp, scp, and piped, in that order [default]
|
||||||
|
#transfer_method = smart
|
||||||
|
|
||||||
# if False, sftp will not use batch mode to transfer files. This may cause some
|
# if False, sftp will not use batch mode to transfer files. This may cause some
|
||||||
# types of file transfer failures impossible to catch however, and should
|
# types of file transfer failures impossible to catch however, and should
|
||||||
# only be disabled if your sftp version has problems with batch mode
|
# only be disabled if your sftp version has problems with batch mode
|
||||||
|
|
|
@ -218,6 +218,7 @@ DEFAULT_VAULT_PASSWORD_FILE = get_config(p, DEFAULTS, 'vault_password_file', 'AN
|
||||||
DEFAULT_TRANSPORT = get_config(p, DEFAULTS, 'transport', 'ANSIBLE_TRANSPORT', 'smart')
|
DEFAULT_TRANSPORT = get_config(p, DEFAULTS, 'transport', 'ANSIBLE_TRANSPORT', 'smart')
|
||||||
DEFAULT_SCP_IF_SSH = get_config(p, 'ssh_connection', 'scp_if_ssh', 'ANSIBLE_SCP_IF_SSH', 'smart')
|
DEFAULT_SCP_IF_SSH = get_config(p, 'ssh_connection', 'scp_if_ssh', 'ANSIBLE_SCP_IF_SSH', 'smart')
|
||||||
DEFAULT_SFTP_BATCH_MODE = get_config(p, 'ssh_connection', 'sftp_batch_mode', 'ANSIBLE_SFTP_BATCH_MODE', True, value_type='boolean')
|
DEFAULT_SFTP_BATCH_MODE = get_config(p, 'ssh_connection', 'sftp_batch_mode', 'ANSIBLE_SFTP_BATCH_MODE', True, value_type='boolean')
|
||||||
|
DEFAULT_SSH_TRANSFER_METHOD = get_config(p, 'ssh_connection', 'transfer_method', 'ANSIBLE_SSH_TRANSFER_METHOD', None)
|
||||||
DEFAULT_MANAGED_STR = get_config(p, DEFAULTS, 'ansible_managed', None, 'Ansible managed')
|
DEFAULT_MANAGED_STR = get_config(p, DEFAULTS, 'ansible_managed', None, 'Ansible managed')
|
||||||
DEFAULT_SYSLOG_FACILITY = get_config(p, DEFAULTS, 'syslog_facility', 'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
|
DEFAULT_SYSLOG_FACILITY = get_config(p, DEFAULTS, 'syslog_facility', 'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
|
||||||
DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', False, value_type='boolean')
|
DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', False, value_type='boolean')
|
||||||
|
|
|
@ -73,6 +73,7 @@ MAGIC_VARIABLE_MAPPING = dict(
|
||||||
sftp_extra_args = ('ansible_sftp_extra_args',),
|
sftp_extra_args = ('ansible_sftp_extra_args',),
|
||||||
scp_extra_args = ('ansible_scp_extra_args',),
|
scp_extra_args = ('ansible_scp_extra_args',),
|
||||||
ssh_extra_args = ('ansible_ssh_extra_args',),
|
ssh_extra_args = ('ansible_ssh_extra_args',),
|
||||||
|
ssh_transfer_method = ('ansible_ssh_transfer_method',),
|
||||||
sudo = ('ansible_sudo',),
|
sudo = ('ansible_sudo',),
|
||||||
sudo_user = ('ansible_sudo_user',),
|
sudo_user = ('ansible_sudo_user',),
|
||||||
sudo_pass = ('ansible_sudo_password', 'ansible_sudo_pass'),
|
sudo_pass = ('ansible_sudo_password', 'ansible_sudo_pass'),
|
||||||
|
@ -173,6 +174,7 @@ class PlayContext(Base):
|
||||||
_scp_extra_args = FieldAttribute(isa='string')
|
_scp_extra_args = FieldAttribute(isa='string')
|
||||||
_ssh_extra_args = FieldAttribute(isa='string')
|
_ssh_extra_args = FieldAttribute(isa='string')
|
||||||
_ssh_executable = FieldAttribute(isa='string', default=C.ANSIBLE_SSH_EXECUTABLE)
|
_ssh_executable = FieldAttribute(isa='string', default=C.ANSIBLE_SSH_EXECUTABLE)
|
||||||
|
_ssh_transfer_method = FieldAttribute(isa='string', default=C.DEFAULT_SSH_TRANSFER_METHOD)
|
||||||
_connection_lockfd= FieldAttribute(isa='int')
|
_connection_lockfd= FieldAttribute(isa='int')
|
||||||
_pipelining = FieldAttribute(isa='bool', default=C.ANSIBLE_SSH_PIPELINING)
|
_pipelining = FieldAttribute(isa='bool', default=C.ANSIBLE_SSH_PIPELINING)
|
||||||
_accelerate = FieldAttribute(isa='bool', default=False)
|
_accelerate = FieldAttribute(isa='bool', default=False)
|
||||||
|
|
|
@ -34,7 +34,7 @@ from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNo
|
||||||
from ansible.errors import AnsibleOptionsError
|
from ansible.errors import AnsibleOptionsError
|
||||||
from ansible.module_utils.basic import BOOLEANS
|
from ansible.module_utils.basic import BOOLEANS
|
||||||
from ansible.module_utils._text import to_bytes, to_native, to_text
|
from ansible.module_utils._text import to_bytes, to_native, to_text
|
||||||
from ansible.plugins.connection import ConnectionBase
|
from ansible.plugins.connection import ConnectionBase, BUFSIZE
|
||||||
from ansible.utils.path import unfrackpath, makedirs_safe
|
from ansible.utils.path import unfrackpath, makedirs_safe
|
||||||
|
|
||||||
boolean = C.mk_boolean
|
boolean = C.mk_boolean
|
||||||
|
@ -605,44 +605,70 @@ class Connection(ConnectionBase):
|
||||||
# accept them for hostnames and IPv4 addresses too.
|
# accept them for hostnames and IPv4 addresses too.
|
||||||
host = '[%s]' % self.host
|
host = '[%s]' % self.host
|
||||||
|
|
||||||
# since this can be a non-bool now, we need to handle it correctly
|
# Transfer methods to try
|
||||||
scp_if_ssh = C.DEFAULT_SCP_IF_SSH
|
methods = []
|
||||||
if not isinstance(scp_if_ssh, bool):
|
|
||||||
scp_if_ssh = scp_if_ssh.lower()
|
|
||||||
if scp_if_ssh in BOOLEANS:
|
|
||||||
scp_if_ssh = boolean(scp_if_ssh)
|
|
||||||
elif scp_if_ssh != 'smart':
|
|
||||||
raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]')
|
|
||||||
|
|
||||||
# create a list of commands to use based on config options
|
# Use the transfer_method option if set, otherwise use scp_if_ssh
|
||||||
methods = ['sftp']
|
ssh_transfer_method = self._play_context.ssh_transfer_method
|
||||||
if scp_if_ssh == 'smart':
|
if ssh_transfer_method is not None:
|
||||||
methods.append('scp')
|
if not (ssh_transfer_method in ('smart', 'sftp', 'scp', 'piped')):
|
||||||
elif scp_if_ssh:
|
raise AnsibleOptionsError('transfer_method needs to be one of [smart|sftp|scp|piped]')
|
||||||
methods = ['scp']
|
if ssh_transfer_method == 'smart':
|
||||||
|
methods = ['sftp', 'scp', 'piped']
|
||||||
|
else:
|
||||||
|
methods = [ssh_transfer_method]
|
||||||
|
else:
|
||||||
|
# since this can be a non-bool now, we need to handle it correctly
|
||||||
|
scp_if_ssh = C.DEFAULT_SCP_IF_SSH
|
||||||
|
if not isinstance(scp_if_ssh, bool):
|
||||||
|
scp_if_ssh = scp_if_ssh.lower()
|
||||||
|
if scp_if_ssh in BOOLEANS:
|
||||||
|
scp_if_ssh = boolean(scp_if_ssh)
|
||||||
|
elif scp_if_ssh != 'smart':
|
||||||
|
raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]')
|
||||||
|
if scp_if_ssh == 'smart':
|
||||||
|
methods = ['sftp', 'scp', 'piped']
|
||||||
|
elif scp_if_ssh == True:
|
||||||
|
methods = ['scp']
|
||||||
|
else:
|
||||||
|
methods = ['sftp']
|
||||||
|
|
||||||
success = False
|
success = False
|
||||||
res = None
|
res = None
|
||||||
for method in methods:
|
for method in methods:
|
||||||
|
returncode = stdout = stderr = None
|
||||||
if method == 'sftp':
|
if method == 'sftp':
|
||||||
cmd = self._build_command('sftp', to_bytes(host))
|
cmd = self._build_command('sftp', to_bytes(host))
|
||||||
in_data = u"{0} {1} {2}\n".format(sftp_action, shlex_quote(in_path), shlex_quote(out_path))
|
in_data = u"{0} {1} {2}\n".format(sftp_action, shlex_quote(in_path), shlex_quote(out_path))
|
||||||
|
in_data = to_bytes(in_data, nonstring='passthru')
|
||||||
|
(returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
|
||||||
elif method == 'scp':
|
elif method == 'scp':
|
||||||
if sftp_action == 'get':
|
if sftp_action == 'get':
|
||||||
cmd = self._build_command('scp', u'{0}:{1}'.format(host, shlex_quote(in_path)), out_path)
|
cmd = self._build_command('scp', u'{0}:{1}'.format(host, shlex_quote(in_path)), out_path)
|
||||||
else:
|
else:
|
||||||
cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, shlex_quote(out_path)))
|
cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, shlex_quote(out_path)))
|
||||||
in_data = None
|
in_data = None
|
||||||
|
(returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
|
||||||
|
elif method == 'piped':
|
||||||
|
if sftp_action == 'get':
|
||||||
|
# we pass sudoable=False to disable pty allocation, which
|
||||||
|
# would end up mixing stdout/stderr and screwing with newlines
|
||||||
|
(returncode, stdout, stderr) = self._exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE), sudoable=False)
|
||||||
|
out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+')
|
||||||
|
out_file.write(stdout)
|
||||||
|
out_file.close()
|
||||||
|
else:
|
||||||
|
in_data = open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb').read()
|
||||||
|
in_data = to_bytes(in_data, nonstring='passthru')
|
||||||
|
(returncode, stdout, stderr) = self._exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), in_data=in_data)
|
||||||
|
|
||||||
in_data = to_bytes(in_data, nonstring='passthru')
|
|
||||||
(returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
|
|
||||||
# Check the return code and rollover to next method if failed
|
# Check the return code and rollover to next method if failed
|
||||||
if returncode == 0:
|
if returncode == 0:
|
||||||
success = True
|
success = True
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
# If not in smart mode, the data will be printed by the raise below
|
# If not in smart mode, the data will be printed by the raise below
|
||||||
if scp_if_ssh == 'smart':
|
if len(methods) > 1:
|
||||||
display.warning(msg='%s transfer mechanism failed on %s. Use ANSIBLE_DEBUG=1 to see detailed information' % (method, host))
|
display.warning(msg='%s transfer mechanism failed on %s. Use ANSIBLE_DEBUG=1 to see detailed information' % (method, host))
|
||||||
display.debug(msg='%s' % to_native(stdout))
|
display.debug(msg='%s' % to_native(stdout))
|
||||||
display.debug(msg='%s' % to_native(stderr))
|
display.debug(msg='%s' % to_native(stderr))
|
||||||
|
|
|
@ -6,3 +6,5 @@ set -eux
|
||||||
./posix.sh "$@"
|
./posix.sh "$@"
|
||||||
# scp
|
# scp
|
||||||
ANSIBLE_SCP_IF_SSH=true ./posix.sh "$@"
|
ANSIBLE_SCP_IF_SSH=true ./posix.sh "$@"
|
||||||
|
# piped
|
||||||
|
ANSIBLE_SSH_TRANSFER_METHOD=piped ./posix.sh "$@"
|
||||||
|
|
Loading…
Reference in a new issue