Fix prompt mismatch issue for ios (#47004)

* Fix prompt mismatch issue for ios

Fixes #40884 #44463 #46082

*  If the command prompt is matched check if data is
   still pending to be read from buffer.
*  This fix adds a new timer `buffer_read_timeout`
   which will be trigerred after command prompt
   is matched and data is attempted to be read from channel.
   If not data is present of channel the timer will expire
   and response we be returned to calling function.

* Update doc

* Fix review comments

* Update changelog

* Fix unit test CI failure

(cherry picked from commit 335a979f1d)
This commit is contained in:
Ganesh Nalawade 2018-10-22 21:05:15 +05:30 committed by Toshio Kuratomi
parent 2d39ad036e
commit 47f2352d33
4 changed files with 114 additions and 8 deletions

View file

@ -0,0 +1,2 @@
bugfixes:
- Fix prompt mismatch issue for ios (https://github.com/ansible/ansible/issues/47004)

View file

@ -656,3 +656,38 @@ Example Ansible inventory file
This is done to prevent secrets from leaking out, for example in ``ps`` output. This is done to prevent secrets from leaking out, for example in ``ps`` output.
We recommend using SSH Keys, and if needed an ssh-agent, rather than passwords, where ever possible. We recommend using SSH Keys, and if needed an ssh-agent, rather than passwords, where ever possible.
Miscellaneous Issues
====================
Intermittent failure while using ``network_cli`` connection type
----------------------------------------------------------------
If the command prompt received in response is not matched correctly within
the ``network_cli`` connection plugin the task might fail intermittently with truncated
response or with the error message ``operation requires privilege escalation``.
Starting in 2.7.1 a new buffer read timer is added to ensure prompts are matched properly
and a complete response is send in output. The timer default value is 0.1 seconds and
can be adjusted on a per task basis or can be set globally in seconds.
Example Per task timer setting
.. code-block:: yaml
- name: gather ios facts
ios_facts:
gather_subset: all
register: result
vars:
ansible_buffer_read_timeout: 2
To make this a global setting, add the following to your ``ansible.cfg`` file:
.. code-block:: ini
[persistent_connection]
buffer_read_timeout = 2
This timer delay per command executed on remote host can be disabled by setting the value to zero.

View file

@ -157,6 +157,21 @@ options:
- name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT - name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT
vars: vars:
- name: ansible_command_timeout - name: ansible_command_timeout
persistent_buffer_read_timeout:
type: float
description:
- Configures, in seconds, the amount of time to wait for the data to be read
from Paramiko channel after the command prompt is matched. This timeout
value ensures that command prompt matched is correct and there is no more data
left to be received from remote host.
default: 0.1
ini:
- section: persistent_connection
key: buffer_read_timeout
env:
- name: ANSIBLE_PERSISTENT_BUFFER_READ_TIMEOUT
vars:
- name: ansible_buffer_read_timeout
""" """
import getpass import getpass
@ -164,6 +179,7 @@ import json
import logging import logging
import re import re
import os import os
import signal
import socket import socket
import traceback import traceback
@ -183,6 +199,10 @@ except ImportError:
display = Display() display = Display()
class AnsibleCmdRespRecv(Exception):
pass
class Connection(NetworkConnectionBase): class Connection(NetworkConnectionBase):
''' CLI (shell) SSH connections on Paramiko ''' ''' CLI (shell) SSH connections on Paramiko '''
@ -199,6 +219,7 @@ class Connection(NetworkConnectionBase):
self._matched_pattern = None self._matched_pattern = None
self._last_response = None self._last_response = None
self._history = list() self._history = list()
self._command_response = None
self._terminal = None self._terminal = None
self.cliconf = None self.cliconf = None
@ -342,15 +363,39 @@ class Connection(NetworkConnectionBase):
''' '''
Handles receiving of output from command Handles receiving of output from command
''' '''
recv = BytesIO()
handled = False
self._matched_prompt = None self._matched_prompt = None
self._matched_cmd_prompt = None self._matched_cmd_prompt = None
recv = BytesIO()
handled = False
command_prompt_matched = False
matched_prompt_window = window_count = 0 matched_prompt_window = window_count = 0
command_timeout = self.get_option('persistent_command_timeout')
self._validate_timeout_value(command_timeout, "persistent_command_timeout")
buffer_read_timeout = self.get_option('persistent_buffer_read_timeout')
self._validate_timeout_value(buffer_read_timeout, "persistent_buffer_read_timeout")
while True: while True:
data = self._ssh_shell.recv(256) if command_prompt_matched:
try:
signal.signal(signal.SIGALRM, self._handle_buffer_read_timeout)
signal.setitimer(signal.ITIMER_REAL, buffer_read_timeout)
data = self._ssh_shell.recv(256)
signal.alarm(0)
# if data is still received on channel it indicates the prompt string
# is wrongly matched in between response chunks, continue to read
# remaining response.
command_prompt_matched = False
# restart command_timeout timer
signal.signal(signal.SIGALRM, self._handle_command_timeout)
signal.alarm(command_timeout)
except AnsibleCmdRespRecv:
return self._command_response
else:
data = self._ssh_shell.recv(256)
# when a channel stream is closed, received data will be empty # when a channel stream is closed, received data will be empty
if not data: if not data:
@ -376,7 +421,11 @@ class Connection(NetworkConnectionBase):
if self._find_prompt(window): if self._find_prompt(window):
self._last_response = recv.getvalue() self._last_response = recv.getvalue()
resp = self._strip(self._last_response) resp = self._strip(self._last_response)
return self._sanitize(resp, command) self._command_response = self._sanitize(resp, command)
if buffer_read_timeout == 0.0:
return self._command_response
else:
command_prompt_matched = True
def send(self, command, prompt=None, answer=None, newline=True, sendonly=False, prompt_retry_check=False, check_all=False): def send(self, command, prompt=None, answer=None, newline=True, sendonly=False, prompt_retry_check=False, check_all=False):
''' '''
@ -398,6 +447,17 @@ class Connection(NetworkConnectionBase):
display.vvvv(traceback.format_exc(), host=self._play_context.remote_addr) display.vvvv(traceback.format_exc(), host=self._play_context.remote_addr)
raise AnsibleConnectionFailure("timeout trying to send command: %s" % command.strip()) raise AnsibleConnectionFailure("timeout trying to send command: %s" % command.strip())
def _handle_buffer_read_timeout(self, signum, frame):
display.vvvv("Response received, triggered 'persistent_buffer_read_timeout' timer of %s seconds"
% self.get_option('persistent_buffer_read_timeout'), host=self._play_context.remote_addr)
raise AnsibleCmdRespRecv()
def _handle_command_timeout(self, signum, frame):
msg = 'command timeout triggered, timeout value is %s secs.\nSee the timeout setting options in the Network Debug and Troubleshooting Guide.'\
% self.get_option('persistent_command_timeout')
display.display(msg, log_only=True)
raise AnsibleConnectionFailure(msg)
def _strip(self, data): def _strip(self, data):
''' '''
Removes ANSI codes from device response Removes ANSI codes from device response
@ -488,3 +548,7 @@ class Connection(NetworkConnectionBase):
raise AnsibleConnectionFailure(errored_response) raise AnsibleConnectionFailure(errored_response)
return False return False
def _validate_timeout_value(self, timeout, timer_name):
if timeout < 0:
raise AnsibleConnectionFailure("'%s' timer value '%s' is invalid, value should be greater than or equal to zero." % (timer_name, timeout))

View file

@ -21,6 +21,7 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import re import re
import time
import json import json
from io import StringIO from io import StringIO
@ -28,6 +29,7 @@ from io import StringIO
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock from ansible.compat.tests.mock import patch, MagicMock
from ansible.module_utils._text import to_text
from ansible.errors import AnsibleConnectionFailure from ansible.errors import AnsibleConnectionFailure
from ansible.playbook.play_context import PlayContext from ansible.playbook.play_context import PlayContext
from ansible.plugins.connection import network_cli from ansible.plugins.connection import network_cli
@ -140,15 +142,18 @@ class TestConnectionClass(unittest.TestCase):
device# device#
""" """
mock__shell.recv.return_value = response mock__shell.recv.side_effect = [response, None]
conn.get_option = MagicMock()
conn.get_option.return_value = 1
output = conn.send(b'command', None, None, None) output = conn.send(b'command', None, None, None)
mock__shell.sendall.assert_called_with(b'command\r') mock__shell.sendall.assert_called_with(b'command\r')
self.assertEqual(output, 'command response') self.assertEqual(to_text(conn._command_response), 'command response')
mock__shell.reset_mock() mock__shell.reset_mock()
mock__shell.recv.return_value = b"ERROR: error message device#" mock__shell.recv.side_effect = [b"ERROR: error message device#"]
with self.assertRaises(AnsibleConnectionFailure) as exc: with self.assertRaises(AnsibleConnectionFailure) as exc:
conn.send(b'command', None, None, None) conn.send(b'command', None, None, None)