From 8ef2e6da05333f49988949f973e627b582a27beb Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Fri, 8 Mar 2019 10:38:02 +1000 Subject: [PATCH] Add support for Windows hosts in the SSH connection plugin (#47732) * Add support for Windows hosts in the SSH connection plugin * fix Python 2.6 unit test and sanity issues * fix up connection tests in CI, disable SCP for now * ensure we don't pollute the existing environment during the test * Add connection_windows_ssh to classifier * use test dir for inventory file * Required powershell as default shell and fix tests * Remove exlicit become_methods on connection * clarify console encoding comment * ignore recent SCP errors in integration tests * Add cmd shell type and added more tests * Fix some doc issues * revises windows faq * add anchors for windows links * revises windows setup page * Update changelogs/fragments/windows-ssh.yaml Co-Authored-By: jborean93 --- changelogs/fragments/windows-ssh.yaml | 2 + docs/docsite/rst/user_guide/windows.rst | 4 +- docs/docsite/rst/user_guide/windows_faq.rst | 109 +++++++----- docs/docsite/rst/user_guide/windows_setup.rst | 160 +++++++++++++++--- lib/ansible/plugins/action/__init__.py | 2 +- lib/ansible/plugins/action/script.py | 6 +- .../plugins/action/wait_for_connection.py | 2 +- lib/ansible/plugins/connection/__init__.py | 1 + lib/ansible/plugins/connection/ssh.py | 61 ++++++- lib/ansible/plugins/connection/winrm.py | 19 +-- .../plugins/doc_fragments/shell_windows.py | 47 +++++ lib/ansible/plugins/shell/__init__.py | 4 + lib/ansible/plugins/shell/cmd.py | 57 +++++++ lib/ansible/plugins/shell/powershell.py | 77 ++++----- .../targets/connection_windows_ssh/aliases | 6 + .../targets/connection_windows_ssh/runme.sh | 54 ++++++ .../test_connection.inventory.j2 | 12 ++ .../targets/connection_windows_ssh/tests.yml | 32 ++++ .../connection_windows_ssh/tests_fetch.yml | 41 +++++ .../targets/connection_windows_ssh/windows.sh | 25 +++ test/runner/lib/classification.py | 6 + test/runner/lib/executor.py | 4 + test/units/plugins/shell/test_cmd.py | 16 ++ test/units/plugins/shell/test_powershell.py | 53 ++++++ 24 files changed, 657 insertions(+), 143 deletions(-) create mode 100644 changelogs/fragments/windows-ssh.yaml create mode 100644 lib/ansible/plugins/doc_fragments/shell_windows.py create mode 100644 lib/ansible/plugins/shell/cmd.py create mode 100644 test/integration/targets/connection_windows_ssh/aliases create mode 100755 test/integration/targets/connection_windows_ssh/runme.sh create mode 100644 test/integration/targets/connection_windows_ssh/test_connection.inventory.j2 create mode 100644 test/integration/targets/connection_windows_ssh/tests.yml create mode 100644 test/integration/targets/connection_windows_ssh/tests_fetch.yml create mode 100755 test/integration/targets/connection_windows_ssh/windows.sh create mode 100644 test/units/plugins/shell/test_cmd.py create mode 100644 test/units/plugins/shell/test_powershell.py diff --git a/changelogs/fragments/windows-ssh.yaml b/changelogs/fragments/windows-ssh.yaml new file mode 100644 index 00000000000..be31d70e041 --- /dev/null +++ b/changelogs/fragments/windows-ssh.yaml @@ -0,0 +1,2 @@ +minor_changes: +- Added experimental support for connecting to Windows hosts over SSH using ``ansible_shell_type=cmd`` or ``ansible_shell_type=powershell`` diff --git a/docs/docsite/rst/user_guide/windows.rst b/docs/docsite/rst/user_guide/windows.rst index 6635383e499..4e01c3a44e3 100644 --- a/docs/docsite/rst/user_guide/windows.rst +++ b/docs/docsite/rst/user_guide/windows.rst @@ -1,3 +1,5 @@ +.. _windows: + Windows Guides `````````````` @@ -5,7 +7,7 @@ The following sections provide information on managing Windows hosts with Ansible. Because Windows is a non-POSIX-compliant operating system, there are differences between -how Ansible interacts with them and the way Windows works. These guides will highlight +how Ansible interacts with them and the way Windows works. These guides will highlight some of the differences between Linux/Unix hosts and hosts running Windows. .. toctree:: diff --git a/docs/docsite/rst/user_guide/windows_faq.rst b/docs/docsite/rst/user_guide/windows_faq.rst index 68f82aa1b4a..a45ea50f6ea 100644 --- a/docs/docsite/rst/user_guide/windows_faq.rst +++ b/docs/docsite/rst/user_guide/windows_faq.rst @@ -1,3 +1,5 @@ +.. _windows_faq: + Windows Frequently Asked Questions ================================== @@ -6,7 +8,7 @@ their answers. .. note:: This document covers questions about managing Microsoft Windows servers with Ansible. For questions about Ansible Core, please see the - :ref:`FAQ page `. + :ref:`general FAQ page `. Does Ansible work with Windows XP or Server 2003? `````````````````````````````````````````````````` @@ -22,19 +24,19 @@ supported operating system versions are: * Windows 8.1 * Windows 10 -Ansible also has minimum PowerShell version requirements - please see -:doc:`windows_setup` for the latest information. +Ansible also has minimum PowerShell version requirements - please see +:ref:`windows_setup` for the latest information. -Can I Manage Windows Nano Server? -````````````````````````````````` +Can I manage Windows Nano Server with Ansible? +`````````````````````````````````````````````` Windows Nano Server is not currently supported by Ansible, since it does not have access to the full .NET Framework that is used by the majority of the modules and internal components. Can Ansible run on Windows? ``````````````````````````` -No, Ansible cannot run on a Windows host natively and can only manage Windows hosts, -but Ansible can be run under the Windows Subsystem for Linux (WSL). +No, Ansible can only manage Windows hosts. Ansible cannot run on a Windows host +natively, though it can run under the Windows Subsystem for Linux (WSL). .. note:: The Windows Subsystem for Linux is not supported by Ansible and should not be used for production systems. @@ -60,16 +62,20 @@ installed version and then clone the git repo. # To enable Ansible on login, run the following echo ". ~/ansible/hacking/env-setup -q' >> ~/.bashrc -Can I use SSH keys to authenticate? -``````````````````````````````````` -Windows uses WinRM as the transport protocol. WinRM supports a wide range of -authentication options. The closet option to SSH keys is to use the certificate -authentication option which maps an X509 certificate to a local user. +Can I use SSH keys to authenticate to Windows hosts? +```````````````````````````````````````````````````` +SSH keys are not supported when using the WinRM or PSRP connection plugins. +These connection plugins support X509 certificates for authentication instead +of the SSH key pairs that SSH supports. -The way that these certificates are generated and mapped to a user is different -from the SSH implementation; consult the :doc:`windows_winrm` documentation for +The way X509 certificates are generated and mapped to a user is different +from the SSH implementation; consult the :ref:`windows_winrm` documentation for more information. +Ansible 2.8 has added experimental support for using the SSH connection plugin, +which supports authentication with SSH keys, to connect to Windows servers. See `this question ` +for more information. + .. _windows_faq_winrm: Why can I run a command locally that does not work under Ansible? @@ -82,7 +88,7 @@ running a command locally in these ways: delegate the user's credentials to a network resource, causing ``Access is Denied`` errors. -* All processes run under WinRM are in a non-interactive session. Applications +* All processes run under WinRM are in a non-interactive session. Applications that require an interactive session will not work. * When running through WinRM, Windows restricts access to internal Windows @@ -93,7 +99,7 @@ Some ways to bypass these restrictions are to: * Use ``become``, which runs a command as it would when run locally. This will bypass most WinRM restrictions, as Windows is unaware the process is running - under WinRM when ``become`` is used. See the :doc:`become` documentation for more + under WinRM when ``become`` is used. See the :ref:`become` documentation for more information. * Use a scheduled task, which can be created with ``win_scheduled_task``. Like @@ -107,15 +113,15 @@ Some ways to bypass these restrictions are to: authentication option that supports credential delegation can be used. Both CredSSP and Kerberos with credential delegation enabled can support this. -See :doc:`become` more info on how to use become. The limitations section at -:doc:`windows_winrm` has more details around WinRM limitations. +See :ref:`become` more info on how to use become. The limitations section at +:ref:`windows_winrm` has more details around WinRM limitations. -This program won't install with Ansible -``````````````````````````````````````` +This program won't install on Windows with Ansible +`````````````````````````````````````````````````` See :ref:`this question ` for more information about WinRM limitations. -What modules are available? -``````````````````````````` +What Windows modules are available? +``````````````````````````````````` Most of the Ansible modules in Ansible Core are written for a combination of Linux/Unix machines and arbitrary web services. These modules are written in Python and most of them do not work on Windows. @@ -147,39 +153,54 @@ In addition, the following Ansible Core modules/action-plugins work with Windows * template (also: win_template) * wait_for_connection -Can I run Python modules? -````````````````````````` +Can I run Python modules on Windows hosts? +`````````````````````````````````````````` No, the WinRM connection protocol is set to use PowerShell modules, so Python modules will not work. A way to bypass this issue to use ``delegate_to: localhost`` to run a Python module on the Ansible controller. This is useful if during a playbook, an external service needs to be contacted and there is no equivalent Windows module available. -Can I connect over SSH? -``````````````````````` -Microsoft has announced and is developing a fork of OpenSSH for Windows that -allows remote manage of Windows servers through the SSH protocol instead of -WinRM. While this can be installed and used right now for normal SSH clients, -it is still in beta from Microsoft and the required functionality has not been -developed within Ansible yet. +.. _winrm_faq_ssh: -There are future plans on adding this feature and this page will be updated -once more information can be shared. +Can I connect to Windows hosts over SSH? +```````````````````````````````````````` +Ansible 2.8 has added experimental support for using the SSH connection plugin +to manage Windows hosts. To connect to Windows hosts over SSH, you must install and configure the `Win32-OpenSSH `_ +fork that is in development with Microsoft on +the Windows host(s). While most of the basics should work with SSH, +``Win32-OpenSSH`` is rapidly changing, with new features added and bugs +fixed in every release. It is highly recommend you install the latest release +of ``Win32-OpenSSH`` from the GitHub Releases page when using it with Ansible +on Windows hosts. -Why is connecting to the host via ssh failing? -`````````````````````````````````````````````` -When trying to connect to a Windows host and the output error indicates that -SSH was used, then this is an indication that the connection vars are not set -properly or the host is not inheriting them correctly. +To use SSH as the connection to a Windows host, set the following variables in +the inventory:: + + ansible_connection=ssh + + # Set either cmd or powershell not both + ansible_shell_type=cmd + # ansible_shell_type=powershell + +The value for ``ansible_shell_type`` should either be ``cmd`` or ``powershell``. +Use ``cmd`` if the ``DefaultShell`` has not been configured on the SSH service +and ``powershell`` if that has been set as the ``DefaultShell``. + +Why is connecting to a Windows host via SSH failing? +```````````````````````````````````````````````````` +Unless you are using ``Win32-OpenSSH`` as described above, you must connect to +Windows hosts using :ref:`windows_winrm`. If your Ansible output indicates that +SSH was used, either you did not set the connection vars properly or the host is not inheriting them correctly. Make sure ``ansible_connection: winrm`` is set in the inventory for the Windows -host. +host(s). Why are my credentials being rejected? `````````````````````````````````````` This can be due to a myriad of reasons unrelated to incorrect credentials. -See HTTP 401/Credentials Rejected at :doc:`windows_setup` for a more detailed +See HTTP 401/Credentials Rejected at :ref:`windows_setup` for a more detailed guide of this could mean. Why am I getting an error SSL CERTIFICATE_VERIFY_FAILED? @@ -196,13 +217,11 @@ host. .. seealso:: - :doc:`index` - The documentation index - :doc:`windows` + :ref:`windows` The Windows documentation index - :doc:`playbooks` + :ref:`about_playbooks` An introduction to playbooks - :doc:`playbooks_best_practices` + :ref:`playbooks_best_practices` Best practices advice `User Mailing List `_ Have a question? Stop by the google group! diff --git a/docs/docsite/rst/user_guide/windows_setup.rst b/docs/docsite/rst/user_guide/windows_setup.rst index 965405e4e0d..fcdfa129005 100644 --- a/docs/docsite/rst/user_guide/windows_setup.rst +++ b/docs/docsite/rst/user_guide/windows_setup.rst @@ -1,8 +1,10 @@ +.. _windows_setup: + Setting up a Windows Host ========================= This document discusses the setup that is required before Ansible can communicate with a Microsoft Windows host. -.. contents:: Topics +.. contents:: :local: Host Requirements @@ -12,7 +14,7 @@ Windows host must meet the following requirements: * Ansible's supported Windows versions generally match those under current and extended support from Microsoft. Supported desktop OSs include - Windows 7, 8.1, and 10, and supported server OSs are Windows Server 2008, + Windows 7, 8.1, and 10, and supported server OSs are Windows Server 2008, 2008 R2, 2012, 2012 R2, and 2016. * Ansible requires PowerShell 3.0 or newer and at least .NET 4.0 to be @@ -21,7 +23,7 @@ Windows host must meet the following requirements: * A WinRM listener should be created and activated. More details for this can be found below. -.. Note:: While these are the base requirements for Ansible connectivity, some Ansible +.. Note:: While these are the base requirements for Ansible connectivity, some Ansible modules have additional requirements, such as a newer OS or PowerShell version. Please consult the module's documentation page to determine whether a host meets those requirements. @@ -60,7 +62,7 @@ do this with the following PowerShell commands: Remove-ItemProperty -Path $reg_winlogon_path -Name DefaultUserName -ErrorAction SilentlyContinue Remove-ItemProperty -Path $reg_winlogon_path -Name DefaultPassword -ErrorAction SilentlyContinue -The script works by checking to see what programs need to be installed +The script works by checking to see what programs need to be installed (such as .NET Framework 4.5.2) and what PowerShell version is required. If a reboot is required and the ``username`` and ``password`` parameters are set, the script will automatically reboot and logon when it comes back up from the @@ -77,15 +79,15 @@ actions are required. .. Note:: Windows Server 2008 can only install PowerShell 3.0; specifying a newer version will result in the script failing. -.. Note:: The ``username`` and ``password`` parameters are stored in plain text - in the registry. Make sure the cleanup commands are run after the script finishes +.. Note:: The ``username`` and ``password`` parameters are stored in plain text + in the registry. Make sure the cleanup commands are run after the script finishes to ensure no credentials are still stored on the host. WinRM Memory Hotfix ------------------- When running on PowerShell v3.0, there is a bug with the WinRM service that limits the amount of memory available to WinRM. Without this hotfix installed, -Ansible will fail to execute certain commands on the Windows host. These +Ansible will fail to execute certain commands on the Windows host. These hotfixes should installed as part of the system bootstrapping or imaging process. The script `Install-WMF3Hotfix.ps1 `_ can be used to install the hotfix on affected hosts. @@ -127,10 +129,10 @@ There are different switches and parameters (like ``-EnableCredSSP`` and ``-ForceNewSSLCert``) that can be set alongside this script. The documentation for these options are located at the top of the script itself. -.. Note:: The ConfigureRemotingForAnsible.ps1 script is intended for training and +.. Note:: The ConfigureRemotingForAnsible.ps1 script is intended for training and development purposes only and should not be used in a - production environment, since it enables settings (like ``Basic`` authentication) - that can be inherently insecure. + production environment, since it enables settings (like ``Basic`` authentication) + that can be inherently insecure. WinRM Listener -------------- @@ -193,7 +195,7 @@ the key options that are useful to understand are: .. comment: Pygments powershell lexer does not support colons (i.e. URLs) .. code-block:: guess - + $thumbprint = "E6CDAA82EEAF2ECE8546E05DB7F3E01AA47D76CE" Get-ChildItem -Path cert:\LocalMachine\My -Recurse | Where-Object { $_.Thumbprint -eq $thumbprint } | Select-Object * @@ -204,12 +206,12 @@ There are three ways to set up a WinRM listener: * Using ``winrm quickconfig`` for HTTP or ``winrm quickconfig -transport:https`` for HTTPS. This is the easiest option to use when running outside of a domain environment and a simple listener is - required. Unlike the other options, this process also has the added benefit of - opening up the Firewall for the ports required and starts the WinRM service. + required. Unlike the other options, this process also has the added benefit of + opening up the Firewall for the ports required and starts the WinRM service. -* Using Group Policy Objects. This is the best way to create a listener when the - host is a member of a domain because the configuration is done automatically - without any user input. For more information on group policy objects, see the +* Using Group Policy Objects. This is the best way to create a listener when the + host is a member of a domain because the configuration is done automatically + without any user input. For more information on group policy objects, see the `Group Policy Objects documentation `_. * Using PowerShell to create the listener with a specific configuration. This @@ -253,7 +255,7 @@ To remove a WinRM listener: WinRM Service Options --------------------- -There are a number of options that can be set to control the behavior of the WinRM service component, +There are a number of options that can be set to control the behavior of the WinRM service component, including authentication options and memory settings. To get an output of the current service configuration options, run the @@ -325,7 +327,7 @@ options are: * ``Service\CertificateThumbprint``: This is the thumbprint of the certificate used to encrypt the TLS channel used with CredSSP authentication. By default - this is empty; a self-signed certificate is generated when the WinRM service + this is empty; a self-signed certificate is generated when the WinRM service starts and is used in the TLS process. * ``Winrs\MaxShellRunTime``: This is the maximum time, in milliseconds, that a @@ -365,10 +367,10 @@ command can be used: Common WinRM Issues ------------------- Because WinRM has a wide range of configuration options, it can be difficult -to setup and configure. Because of this complexity, issues that are shown by Ansible -could in fact be issues with the host setup instead. +to setup and configure. Because of this complexity, issues that are shown by Ansible +could in fact be issues with the host setup instead. -One easy way to determine whether a problem is a host issue is to +One easy way to determine whether a problem is a host issue is to run the following command from another Windows host to connect to the target Windows host: @@ -400,7 +402,7 @@ connection. Some things to check for this are: ``ansible_user`` and ``ansible_password`` * Ensure that the user is a member of the local Administrators group or has been explicitly - granted access (a connection test with the ``winrs`` command can be used to + granted access (a connection test with the ``winrs`` command can be used to rule this out). * Make sure that the authentication option set by ``ansible_winrm_transport`` is enabled under @@ -438,7 +440,7 @@ Ansible is unable to reach the host. Some things to check for include: * Make sure the firewall is not set to block the configured WinRM listener ports * Ensure that a WinRM listener is enabled on the port and path set by the host vars -* Ensure that the ``winrm`` service is running on the Windows host and configured for +* Ensure that the ``winrm`` service is running on the Windows host and configured for automatic start Connection Refused Errors @@ -455,13 +457,117 @@ Sometimes an installer may restart the WinRM or HTTP service and cause this erro best way to deal with this is to use ``win_psexec`` from another Windows host. +Windows SSH Setup +````````````````` +Ansible 2.8 has added experimental support for using SSH to connect to a +Windows host. + +.. warning:: + Use this feature at your own risk! + Using SSH with Windows is experimental, the implementation may make + backwards incompatible changes in feature releases. The server side + components can be unreliable depending on the version that is installed. + +Installing Win32-OpenSSH +------------------------ +The first step to using SSH with Windows is to install the `Win32-OpenSSH `_ +service on the Windows host. Microsoft offers a way to install ``Win32-OpenSSH`` through a Windows +capability but currently the version that is installed through this process is +too old to work with Ansible. To install ``Win32-OpenSSH`` for use with +Ansible, select one of these three installation options: + +* Manually install the service, following the `install instructions `_ + from Microsoft. + +* Use ``win_chocolatey`` to install the service:: + + - name: install the Win32-OpenSSH service + win_chocolatey: + name: openssh + package_params: /SSHServerFeature + state: present + +* Use an existing Ansible Galaxy role like `jborean93.win_openssh `_:: + + # Make sure the role has been downloaded first + ansible-galaxy install jborean93.win_openssh + + # main.yml + - name: install Win32-OpenSSH service + hosts: windows + gather_facts: no + roles: + - role: jborean93.win_openssh + opt_openssh_setup_service: True + +.. note:: ``Win32-OpenSSH`` is still a beta product and is constantly + being updated to include new features and bugfixes. If you are using SSH as + a connection option for Windows, it is highly recommend you install the + latest release from one of the 3 methods above. + +Configuring the Win32-OpenSSH shell +----------------------------------- + +By default ``Win32-OpenSSH`` will use ``cmd.exe`` as a shell. To configure a +different shell, use an Ansible task to define the registry setting:: + + - name: set the default shell to PowerShell + win_regedit: + path: HKLM:\SOFTWARE\OpenSSH + name: DefaultShell + data: C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe + type: string + state: present + + # Or revert the settings back to the default, cmd + - name: set the default shell to cmd + win_regedit: + path: HKLM:\SOFTWARE\OpenSSH + name: DefaultShell + state: absent + +Win32-OpenSSH Authentication +---------------------------- +Win32-OpenSSH authentication with Windows is similar to SSH +authentication on Unix/Linux hosts. You can use a plaintext password or +SSH public key authentication, add public keys to an ``authorized_key`` file +in the ``.ssh`` folder of the user's profile directory, and configure the +service using the ``sshd_config`` file used by the SSH service as you would on +a Unix/Linux host. + +When using SSH key authentication with Ansible, the remote session won't have access to the +user's credentials and will fail when attempting to access a network resource. +This is also known as the double-hop or credential delegation issue. There are +two ways to work around this issue: + +* Use plaintext password auth by setting ``ansible_password`` +* Use ``become`` on the task with the credentials of the user that needs access to the remote resource + +Configuring Ansible for SSH on Windows +-------------------------------------- +To configure Ansible to use SSH for Windows hosts, you must set two connection variables: + +* set ``ansible_connection`` to ``ssh`` +* set ``ansible_shell_type`` to ``cmd`` or ``powershell`` + +The ``ansible_shell_type`` variable should reflect the ``DefaultShell`` +configured on the Windows host. Set to ``cmd`` for the default shell or set to +``powershell`` if the ``DefaultShell`` has been changed to PowerShell. + +Known issues with SSH on Windows +-------------------------------- +Using SSH with Windows is experimental, and we expect to uncover more issues. +Here are the known ones: + +* Win32-OpenSSH versions older than ``v7.9.0.0p1-Beta`` do not work when ``powershell`` is the shell type +* While SCP should work, SFTP is the recommended SSH file transfer mechanism to use when copying or fetching a file + + .. seealso:: - :doc:`index` - The documentation index - :doc:`playbooks` + :ref:`about_playbooks` An introduction to playbooks - :doc:`playbooks_best_practices` + :ref:`playbooks_best_practices` Best practices advice :ref:`List of Windows Modules ` Windows specific module list, all implemented in PowerShell diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index 831e4d99792..2c489b43245 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -461,7 +461,7 @@ class ActionBase(with_metaclass(ABCMeta, object)): if remote_user is None: remote_user = self._get_remote_user() - if self._connection._shell.SHELL_FAMILY == 'powershell': + if getattr(self._connection._shell, "_IS_WINDOWS", False): # This won't work on Powershell as-is, so we'll just completely skip until # we have a need for it, at which point we'll have to do something different. return remote_paths diff --git a/lib/ansible/plugins/action/script.py b/lib/ansible/plugins/action/script.py index 16b9fdf65f5..73a70d5323f 100644 --- a/lib/ansible/plugins/action/script.py +++ b/lib/ansible/plugins/action/script.py @@ -65,11 +65,11 @@ class ActionModule(ActionBase): chdir = self._task.args.get('chdir') if chdir: # Powershell is the only Windows-path aware shell - if self._connection._shell.SHELL_FAMILY == 'powershell' and \ + if getattr(self._connection._shell, "_IS_WINDOWS", False) and \ not self.windows_absolute_path_detection.match(chdir): raise AnsibleActionFail('chdir %s must be an absolute path for a Windows remote node' % chdir) # Every other shell is unix-path-aware. - if self._connection._shell.SHELL_FAMILY != 'powershell' and not chdir.startswith('/'): + if not getattr(self._connection._shell, "_IS_WINDOWS", False) and not chdir.startswith('/'): raise AnsibleActionFail('chdir %s must be an absolute path for a Unix-aware remote node' % chdir) # Split out the script as the first item in raw_params using @@ -126,7 +126,7 @@ class ActionModule(ActionBase): exec_data = None # PowerShell runs the script in a special wrapper to enable things # like become and environment args - if self._connection._shell.SHELL_FAMILY == "powershell": + if getattr(self._connection._shell, "_IS_WINDOWS", False): # FUTURE: use a more public method to get the exec payload pc = self._play_context exec_data = ps_manifest._create_powershell_wrapper( diff --git a/lib/ansible/plugins/action/wait_for_connection.py b/lib/ansible/plugins/action/wait_for_connection.py index 7810052cea5..f085e661f9f 100644 --- a/lib/ansible/plugins/action/wait_for_connection.py +++ b/lib/ansible/plugins/action/wait_for_connection.py @@ -86,7 +86,7 @@ class ActionModule(ActionBase): pass # Use win_ping on winrm/powershell, else use ping - if hasattr(self._connection, '_shell_type') and self._connection._shell_type == 'powershell': + if getattr(self._connection._shell, "_IS_WINDOWS", False): ping_result = self._execute_module(module_name='win_ping', module_args=dict(), task_vars=task_vars) else: ping_result = self._execute_module(module_name='ping', module_args=dict(), task_vars=task_vars) diff --git a/lib/ansible/plugins/connection/__init__.py b/lib/ansible/plugins/connection/__init__.py index a790138a469..5bedc6d1543 100644 --- a/lib/ansible/plugins/connection/__init__.py +++ b/lib/ansible/plugins/connection/__init__.py @@ -8,6 +8,7 @@ __metaclass__ = type import fcntl import os import shlex + from abc import abstractmethod, abstractproperty from functools import wraps diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index de96c3869ab..7b878601306 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -276,6 +276,7 @@ import fcntl import hashlib import os import pty +import re import subprocess import time @@ -294,6 +295,7 @@ from ansible.module_utils.six.moves import shlex_quote from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils.parsing.convert_bool import BOOLEANS, boolean from ansible.plugins.connection import ConnectionBase, BUFSIZE +from ansible.plugins.shell.powershell import _parse_clixml from ansible.utils.display import Display from ansible.utils.path import unfrackpath, makedirs_safe @@ -453,6 +455,15 @@ class Connection(ConnectionBase): self.control_path = C.ANSIBLE_SSH_CONTROL_PATH self.control_path_dir = C.ANSIBLE_SSH_CONTROL_PATH_DIR + # Windows operates differently from a POSIX connection/shell plugin, + # we need to set various properties to ensure SSH on Windows continues + # to work + if getattr(self._shell, "_IS_WINDOWS", False): + self.has_native_async = True + self.always_pipeline_modules = True + self.module_implementation_preferences = ('.ps1', '.exe', '') + self.allow_executable = False + # The connection is created by running ssh/scp/sftp from the exec_command, # put_file, and fetch_file methods, so we don't need to do any connection # management here. @@ -742,6 +753,7 @@ class Connection(ConnectionBase): Starts the command and communicates with it until it ends. ''' + # We don't use _shell.quote as this is run on the controller and independent from the shell plugin chosen display_cmd = list(map(shlex_quote, map(to_text, cmd))) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) @@ -1030,6 +1042,12 @@ class Connection(ConnectionBase): # accept them for hostnames and IPv4 addresses too. host = '[%s]' % self.host + smart_methods = ['sftp', 'scp', 'piped'] + + # Windows does not support dd so we cannot use the piped method + if getattr(self._shell, "_IS_WINDOWS", False): + smart_methods.remove('piped') + # Transfer methods to try methods = [] @@ -1039,7 +1057,7 @@ class Connection(ConnectionBase): if not (ssh_transfer_method in ('smart', 'sftp', 'scp', 'piped')): raise AnsibleOptionsError('transfer_method needs to be one of [smart|sftp|scp|piped]') if ssh_transfer_method == 'smart': - methods = ['sftp', 'scp', 'piped'] + methods = smart_methods else: methods = [ssh_transfer_method] else: @@ -1052,7 +1070,7 @@ class Connection(ConnectionBase): elif scp_if_ssh != 'smart': raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]') if scp_if_ssh == 'smart': - methods = ['sftp', 'scp', 'piped'] + methods = smart_methods elif scp_if_ssh is True: methods = ['scp'] else: @@ -1067,10 +1085,11 @@ class Connection(ConnectionBase): (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False) elif method == 'scp': scp = self.get_option('scp_executable') + if sftp_action == 'get': - cmd = self._build_command(scp, u'{0}:{1}'.format(host, shlex_quote(in_path)), out_path) + cmd = self._build_command(scp, u'{0}:{1}'.format(host, self._shell.quote(in_path)), out_path) else: - cmd = self._build_command(scp, in_path, u'{0}:{1}'.format(host, shlex_quote(out_path))) + cmd = self._build_command(scp, in_path, u'{0}:{1}'.format(host, self._shell.quote(out_path))) in_data = None (returncode, stdout, stderr) = self._bare_run(cmd, in_data, checkrc=False) elif method == 'piped': @@ -1105,6 +1124,16 @@ class Connection(ConnectionBase): raise AnsibleError("failed to transfer file to %s %s:\n%s\n%s" % (to_native(in_path), to_native(out_path), to_native(stdout), to_native(stderr))) + def _escape_win_path(self, path): + """ converts a Windows path to one that's supported by SFTP and SCP """ + # If using a root path then we need to start with / + prefix = "" + if re.match(r'^\w{1}:', path): + prefix = "/" + + # Convert all '\' to '/' + return "%s%s" % (prefix, path.replace("\\", "/")) + # # Main public methods # @@ -1115,6 +1144,18 @@ class Connection(ConnectionBase): display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr) + if getattr(self._shell, "_IS_WINDOWS", False): + # Become method 'runas' is done in the wrapper that is executed, + # need to disable sudoable so the bare_run is not waiting for a + # prompt that will not occur + sudoable = False + + # Make sure our first command is to set the console encoding to + # utf-8, this must be done via chcp to get utf-8 (65001) + cmd_parts = ["chcp.com", "65001", self._shell._SHELL_REDIRECT_ALLNULL, self._shell._SHELL_AND] + cmd_parts.extend(self._shell._encode_script(cmd, as_list=True, strict_mode=False, preserve_rc=False)) + cmd = ' '.join(cmd_parts) + # we can only use tty when we are not pipelining the modules. piping # data into /usr/bin/python inside a tty automatically invokes the # python interactive-mode but the modules are not compatible with the @@ -1134,6 +1175,10 @@ class Connection(ConnectionBase): cmd = self._build_command(*args) (returncode, stdout, stderr) = self._run(cmd, in_data, sudoable=sudoable) + # When running on Windows, stderr may contain CLIXML encoded output + if getattr(self._shell, "_IS_WINDOWS", False) and stderr.startswith(b"#< CLIXML"): + stderr = _parse_clixml(stderr) + return (returncode, stdout, stderr) def put_file(self, in_path, out_path): @@ -1145,6 +1190,9 @@ class Connection(ConnectionBase): if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')): raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_native(in_path))) + if getattr(self._shell, "_IS_WINDOWS", False): + out_path = self._escape_win_path(out_path) + return self._file_transport_command(in_path, out_path, 'put') def fetch_file(self, in_path, out_path): @@ -1153,6 +1201,11 @@ class Connection(ConnectionBase): super(Connection, self).fetch_file(in_path, out_path) display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host) + + # need to add / if path is rooted + if getattr(self._shell, "_IS_WINDOWS", False): + in_path = self._escape_win_path(in_path) + return self._file_transport_command(in_path, out_path, 'get') def reset(self): diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index eda7f6ac16b..7c020b142e6 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -104,7 +104,6 @@ import traceback import json import tempfile import subprocess -import xml.etree.ElementTree as ET HAVE_KERBEROS = False try: @@ -122,6 +121,7 @@ from ansible.module_utils.six.moves.urllib.parse import urlunsplit from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils.six import binary_type, PY3 from ansible.plugins.connection import ConnectionBase +from ansible.plugins.shell.powershell import _parse_clixml from ansible.utils.hashing import secure_hash from ansible.utils.path import makedirs_safe from ansible.utils.display import Display @@ -538,28 +538,15 @@ class Connection(ConnectionBase): result.std_err = to_bytes(result.std_err) # parse just stderr from CLIXML output - if self.is_clixml(result.std_err): + if result.std_err.startswith(b"#< CLIXML"): try: - result.std_err = self.parse_clixml_stream(result.std_err) + result.std_err = _parse_clixml(result.std_err) except Exception: # unsure if we're guaranteed a valid xml doc- use raw output in case of error pass return (result.status_code, result.std_out, result.std_err) - def is_clixml(self, value): - return value.startswith(b"#< CLIXML\r\n") - - # hacky way to get just stdout- not always sure of doc framing here, so use with care - def parse_clixml_stream(self, clixml_doc, stream_name='Error'): - clixml = ET.fromstring(clixml_doc.split(b"\r\n", 1)[-1]) - namespace_match = re.match(r'{(.*)}', clixml.tag) - namespace = "{%s}" % namespace_match.group(1) if namespace_match else "" - - strings = clixml.findall("./%sS" % namespace) - lines = [e.text.replace('_x000D__x000A_', '') for e in strings if e.attrib.get('S') == stream_name] - return to_bytes('\r\n'.join(lines)) - # FUTURE: determine buffer size at runtime via remote winrm config? def _put_file_stdin_iterator(self, in_path, out_path, buffer_size=250000): in_size = os.path.getsize(to_bytes(in_path, errors='surrogate_or_strict')) diff --git a/lib/ansible/plugins/doc_fragments/shell_windows.py b/lib/ansible/plugins/doc_fragments/shell_windows.py new file mode 100644 index 00000000000..e4a8b6c3e39 --- /dev/null +++ b/lib/ansible/plugins/doc_fragments/shell_windows.py @@ -0,0 +1,47 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + + +class ModuleDocFragment(object): + + # Windows shell documentation fragment + # FIXME: set_module_language don't belong here but must be set so they don't fail when someone + # get_option('set_module_language') on this plugin + DOCUMENTATION = """ +options: + async_dir: + description: + - Directory in which ansible will keep async job information. + - Before Ansible 2.8, this was set to C(remote_tmp + "\\.ansible_async"). + default: '%USERPROFILE%\\.ansible_async' + ini: + - section: powershell + key: async_dir + vars: + - name: ansible_async_dir + version_added: '2.8' + remote_tmp: + description: + - Temporary directory to use on targets when copying files to the host. + default: '%TEMP%' + ini: + - section: powershell + key: remote_tmp + vars: + - name: ansible_remote_tmp + set_module_language: + description: + - Controls if we set the locale for moduels when executing on the + target. + - Windows only supports C(no) as an option. + type: bool + default: 'no' + choices: + - 'no' + environment: + description: + - Dictionary of environment variables and their values to use when + executing commands. + type: dict + default: {} +""" diff --git a/lib/ansible/plugins/shell/__init__.py b/lib/ansible/plugins/shell/__init__.py index bd1d62ab202..c747845b4fb 100644 --- a/lib/ansible/plugins/shell/__init__.py +++ b/lib/ansible/plugins/shell/__init__.py @@ -221,3 +221,7 @@ class ShellBase(AnsiblePlugin): def wrap_for_exec(self, cmd): """wrap script execution with any necessary decoration (eg '&' for quoted powershell script paths)""" return cmd + + def quote(self, cmd): + """Returns a shell-escaped string that can be safely used as one token in a shell command line""" + return shlex_quote(cmd) diff --git a/lib/ansible/plugins/shell/cmd.py b/lib/ansible/plugins/shell/cmd.py new file mode 100644 index 00000000000..e475e8ee39a --- /dev/null +++ b/lib/ansible/plugins/shell/cmd.py @@ -0,0 +1,57 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +name: cmd +plugin_type: shell +version_added: '2.8' +short_description: Windows Command Prompt +description: +- Used with the 'ssh' connection plugin and no C(DefaultShell) has been set on the Windows host. +extends_documentation_fragment: +- shell_windows +''' + +import re + +from ansible.plugins.shell.powershell import ShellModule as PSShellModule + +# these are the metachars that have a special meaning in cmd that we want to escape when quoting +_find_unsafe = re.compile(r'[\s\(\)\%\!^\"\<\>\&\|]').search + + +class ShellModule(PSShellModule): + + # Common shell filenames that this plugin handles + COMPATIBLE_SHELLS = frozenset() + # Family of shells this has. Must match the filename without extension + SHELL_FAMILY = 'cmd' + + _SHELL_REDIRECT_ALLNULL = '>nul 2>&1' + _SHELL_AND = '&&' + + # Used by various parts of Ansible to do Windows specific changes + _IS_WINDOWS = True + + def quote(self, s): + # cmd does not support single quotes that the shlex_quote uses. We need to override the quoting behaviour to + # better match cmd.exe. + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + + # Return an empty argument + if not s: + return '""' + + if _find_unsafe(s) is None: + return s + + # Escape the metachars as we are quoting the string to stop cmd from interpreting that metachar. For example + # 'file &whoami.exe' would result in 'file $(whoami.exe)' instead of the literal string + # https://stackoverflow.com/questions/3411771/multiple-character-replace-with-python + for c in '^()%!"<>&|': # '^' must be the first char that we scan and replace + if c in s: + s = s.replace(c, "^" + c) + + return '^"' + s + '^"' diff --git a/lib/ansible/plugins/shell/powershell.py b/lib/ansible/plugins/shell/powershell.py index 7f874002b0b..ee23147cc5d 100644 --- a/lib/ansible/plugins/shell/powershell.py +++ b/lib/ansible/plugins/shell/powershell.py @@ -5,60 +5,26 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type DOCUMENTATION = ''' - name: powershell - plugin_type: shell - version_added: "" - short_description: Windows Powershell - description: - - The only option when using 'winrm' as a connection plugin - options: - async_dir: - description: - - Directory in which ansible will keep async job information. - - Before Ansible 2.8, this was set to C(remote_tmp + "\\.ansible_async"). - default: '%USERPROFILE%\\.ansible_async' - ini: - - section: powershell - key: async_dir - vars: - - name: ansible_async_dir - version_added: '2.8' - remote_tmp: - description: - - Temporary directory to use on targets when copying files to the host. - default: '%TEMP%' - ini: - - section: powershell - key: remote_tmp - vars: - - name: ansible_remote_tmp - set_module_language: - description: - - Controls if we set the locale for moduels when executing on the - target. - - Windows only supports C(no) as an option. - type: bool - default: 'no' - choices: - - 'no' - environment: - description: - - Dictionary of environment variables and their values to use when - executing commands. - type: dict - default: {} +name: powershell +plugin_type: shell +version_added: historical +short_description: Windows PowerShell +description: +- The only option when using 'winrm' or 'psrp' as a connection plugin. +- Can also be used when using 'ssh' as a connection plugin and the C(DefaultShell) has been configured to PowerShell. +extends_documentation_fragment: +- shell_windows ''' -# FIXME: admin_users and set_module_language don't belong here but must be set -# so they don't failk when someone get_option('admin_users') on this plugin import base64 import os import re import shlex import pkgutil +import xml.etree.ElementTree as ET from ansible.errors import AnsibleError -from ansible.module_utils._text import to_text +from ansible.module_utils._text import to_bytes, to_text from ansible.plugins.shell import ShellBase @@ -71,6 +37,21 @@ if _powershell_version: _common_args = ['PowerShell', '-Version', _powershell_version] + _common_args[1:] +def _parse_clixml(data, stream="Error"): + """ + Takes a byte string like '#< CLIXML\r\n nul', '^"C:\\temp\\some ^^^%file^% ^> nul^"'] +]) +def test_quote_args(s, expected): + cmd = ShellModule() + actual = cmd.quote(s) + assert actual == expected diff --git a/test/units/plugins/shell/test_powershell.py b/test/units/plugins/shell/test_powershell.py new file mode 100644 index 00000000000..a73070c6895 --- /dev/null +++ b/test/units/plugins/shell/test_powershell.py @@ -0,0 +1,53 @@ +from ansible.plugins.shell.powershell import _parse_clixml + + +def test_parse_clixml_empty(): + empty = b'#< CLIXML\r\n' + expected = b'' + actual = _parse_clixml(empty) + assert actual == expected + + +def test_parse_clixml_with_progress(): + progress = b'#< CLIXML\r\n' \ + b'System.Management.Automation.PSCustomObjectSystem.Object' \ + b'1Preparing modules for first use.0' \ + b'-1-1Completed-1 ' + expected = b'' + actual = _parse_clixml(progress) + assert actual == expected + + +def test_parse_clixml_single_stream(): + single_stream = b'#< CLIXML\r\n' \ + b'fake : The term \'fake\' is not recognized as the name of a cmdlet. Check _x000D__x000A_' \ + b'the spelling of the name, or if a path was included._x000D__x000A_' \ + b'At line:1 char:1_x000D__x000A_' \ + b'+ fake cmdlet_x000D__x000A_+ ~~~~_x000D__x000A_' \ + b' + CategoryInfo : ObjectNotFound: (fake:String) [], CommandNotFoundException_x000D__x000A_' \ + b' + FullyQualifiedErrorId : CommandNotFoundException_x000D__x000A_ _x000D__x000A_' \ + b'' + expected = b"fake : The term 'fake' is not recognized as the name of a cmdlet. Check \r\n" \ + b"the spelling of the name, or if a path was included.\r\n" \ + b"At line:1 char:1\r\n" \ + b"+ fake cmdlet\r\n" \ + b"+ ~~~~\r\n" \ + b" + CategoryInfo : ObjectNotFound: (fake:String) [], CommandNotFoundException\r\n" \ + b" + FullyQualifiedErrorId : CommandNotFoundException\r\n " + actual = _parse_clixml(single_stream) + assert actual == expected + + +def test_parse_clixml_multiple_streams(): + multiple_stream = b'#< CLIXML\r\n' \ + b'fake : The term \'fake\' is not recognized as the name of a cmdlet. Check _x000D__x000A_' \ + b'the spelling of the name, or if a path was included._x000D__x000A_' \ + b'At line:1 char:1_x000D__x000A_' \ + b'+ fake cmdlet_x000D__x000A_+ ~~~~_x000D__x000A_' \ + b' + CategoryInfo : ObjectNotFound: (fake:String) [], CommandNotFoundException_x000D__x000A_' \ + b' + FullyQualifiedErrorId : CommandNotFoundException_x000D__x000A_ _x000D__x000A_' \ + b'hi info' \ + b'' + expected = b"hi info" + actual = _parse_clixml(multiple_stream, stream="Info") + assert actual == expected