Persistent connection timer changes (#27272)

*  Add command_timeout timer that defines the amount
   of time to wait for a command or RPC call before
   timing out.
*  Remove connect_retries and connect_interval configuration
   varaible and replace it with connect_retry_timeout to control
   the timeout value of connection to local scoket.
*  Make required changes to netowrk action plugins and relevant
   network files in module_utils.
*  Required documentation changes.
This commit is contained in:
Ganesh Nalawade 2017-08-01 23:15:45 +05:30 committed by Chris Alfonso
parent 4dd8f281d6
commit 70ce394840
23 changed files with 154 additions and 73 deletions

View file

@ -178,11 +178,11 @@ class Server():
display.display('shutdown local socket, connection was active for %s secs' % delta, log_only=True)
def connect_timeout(self, signum, frame):
display.display('connect timeout triggered, timeout value is %s secs' % C.PERSISTENT_CONNECT_TIMEOUT, log_only=True)
display.display('persistent connection idle timeout triggered, timeout value is %s secs' % C.PERSISTENT_CONNECT_TIMEOUT, log_only=True)
self.shutdown()
def command_timeout(self, signum, frame):
display.display('commnad timeout triggered, timeout value is %s secs' % self.play_context.timeout, log_only=True)
display.display('command timeout triggered, timeout value is %s secs' % self.play_context.timeout, log_only=True)
self.shutdown()
def handler(self, signum, frame):
@ -214,6 +214,7 @@ class Server():
def do_EXEC(self, data):
cmd = data.split(b'EXEC: ')[1]
display.display('Command executed: %s' % cmd, log_only=True)
return self.connection.exec_command(cmd)
def do_PUT(self, data):
@ -352,17 +353,17 @@ def main():
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
attempts = C.PERSISTENT_CONNECT_RETRIES
while bool(attempts):
connect_retry_timeout = C.PERSISTENT_CONNECT_RETRY_TIMEOUT
while bool(connect_retry_timeout):
try:
sock.connect(socket_path)
break
except socket.error:
time.sleep(C.PERSISTENT_CONNECT_INTERVAL)
attempts -= 1
time.sleep(1)
connect_retry_timeout -= 1
else:
display.display('number of connection attempts exceeded, unable to connect to control socket', pc.remote_addr, pc.remote_user, log_only=True)
display.display('persistent_connect_interval=%s, persistent_connect_retries=%s' % (C.PERSISTENT_CONNECT_INTERVAL, C.PERSISTENT_CONNECT_RETRIES), pc.remote_addr, pc.remote_user, log_only=True)
display.display('connect retry timeout expired, unable to connect to control socket', pc.remote_addr, pc.remote_user, log_only=True)
display.display('persistent_connect_retry_timeout is %s secs' % (C.PERSISTENT_CONNECT_RETRY_TIMEOUT), pc.remote_addr, pc.remote_user, log_only=True)
sys.stderr.write('failed to connect to control socket')
sys.exit(255)

View file

@ -166,7 +166,6 @@ or:
"changed": false,
"failed": true,
"msg": "unable to open shell",
"rc": 255
}
Suggestions to resolve:
@ -301,8 +300,8 @@ For example:
2017-04-04 12:19:05,670 p=18591 u=fred | using connection plugin network_cli
2017-04-04 12:19:06,606 p=18591 u=fred | connecting to host veos01 returned an error
2017-04-04 12:19:06,606 p=18591 u=fred | No authentication methods available
2017-04-04 12:19:35,708 p=18591 u=fred | number of connection attempts exceeded, unable to connect to control socket
2017-04-04 12:19:35,709 p=18591 u=fred | persistent_connect_interval=1, persistent_connect_retries=30
2017-04-04 12:19:35,708 p=18591 u=fred | connect retry timeout expired, unable to connect to control socket
2017-04-04 12:19:35,709 p=18591 u=fred | persistent_connect_retry_timeout is 15 secs
Suggestions to resolve:
@ -328,16 +327,62 @@ Timeout issues
Timeouts
--------
Persistent connection idle timeout:
All network modules support a timeout value that can be set on a per task
basis. The timeout value controls the amount of time in seconds before the
For example:
.. code-block:: yaml
2017-04-04 12:19:05,670 p=18591 u=fred | persistent connection idle timeout triggered, timeout value is 30 secs
Suggestions to resolve:
Increase value of presistent connection idle timeout.
.. code-block:: yaml
export ANSIBLE_PERSISTENT_CONNECT_TIMEOUT=60
To make this a permanent change, add the following to your ``ansible.cfg`` file:
.. code-block:: ini
[persistent_connection]
connect_timeout = 60
Command timeout:
For example:
.. code-block:: yaml
2017-04-04 12:19:05,670 p=18591 u=fred | command timeout triggered, timeout value is 10 secs
Suggestions to resolve:
Options 1:
Increase value of command timeout in configuration file or by setting enviornment variable.
Note: This value should be less than persistent connection idle timeout ie. connect_timeout
.. code-block:: yaml
export ANSIBLE_PERSISTENT_COMMAND_TIMEOUT=30
To make this a permanent change, add the following to your ``ansible.cfg`` file:
.. code-block:: ini
[persistent_connection]
command_timeout = 30
Option 2:
Increase command timeout per task basis. All network modules support a
timeout value that can be set on a per task basis.
The timeout value controls the amount of time in seconds before the
task will fail if the command has not returned.
For example:
.. FIXME: Detail error here
Suggestions to resolve:
.. code-block:: yaml
@ -353,6 +398,33 @@ example is saving the current running config on IOS devices to startup config.
In this case, changing the timeout value form the default 10 seconds to 30
seconds will prevent the task from failing before the command completes
successfully.
Note: This value should be less than persistent connection idle timeout ie. connect_timeout
Persistent socket connect timeout:
For example:
.. code-block:: yaml
2017-04-04 12:19:35,708 p=18591 u=fred | connect retry timeout expired, unable to connect to control socket
2017-04-04 12:19:35,709 p=18591 u=fred | persistent_connect_retry_timeout is 15 secs
Suggestions to resolve:
Increase value of presistent connection idle timeout.
Note: This value should be greater than SSH timeout ie. timeout value under defaults
section in configuration file and less than the value of the persistent
connection idle timeout (connect_timeout)
.. code-block:: yaml
export ANSIBLE_PERSISTENT_CONNECT_RETRY_TIMEOUT=30
To make this a permanent change, add the following to your ``ansible.cfg`` file:
.. code-block:: ini
[persistent_connection]
connect_retry_timeout = 30
@ -403,7 +475,6 @@ For example:
"changed": false,
"failed": true,
"msg": "unable to enter configuration mode",
"rc": 255
}
Suggestions to resolve:
@ -461,8 +532,8 @@ Add `authorize: yes` to the task. For example:
.. code-block:: yaml
less $ANSIBLE_LOG_PATH
2017-03-10 15:32:06,173 p=19677 u=fred | number of connection attempts exceeded, unable to connect to control socket
2017-03-10 15:32:06,174 p=19677 u=fred | persistent_connect_interval=1, persistent_connect_retries=10
2017-03-10 15:32:06,173 p=19677 u=fred | connect retry timeout expired, unable to connect to control socket
2017-03-10 15:32:06,174 p=19677 u=fred | persistent_connect_retry_timeout is 15 secs
2017-03-10 15:32:06,222 p=19669 u=fred | fatal: [veos01]: FAILED! => {
Suggestions to resolve:

View file

@ -400,20 +400,21 @@
# Configures the persistent connection timeout value in seconds. This value is
# how long the persistent connection will remain idle before it is destroyed.
# If the connection doesn't receive a request before the timeout value
# expires, the connection is shutdown. The default value is 30 seconds.
# expires, the connection is shutdown. The default value is 30 seconds.
#connect_timeout = 30
# Configures the persistent connection retries. This value configures the
# number of attempts the ansible-connection will make when trying to connect
# to the local domain socket. The default value is 30.
#connect_retries = 30
# Configures the persistent connection retry timeout. This value configures the
# the retry timeout that ansible-connection will wait to connect
# to the local domain socket. This value must be larger than the
# ssh timeout (timeout) and less than persistent connection idle timeout (connect_timeout).
# The default value is 15 seconds.
#connect_retry_timeout = 15
# Configures the amount of time in seconds to wait between connection attempts
# to the local unix domain socket. This value works in conjunction with the
# connect_retries value to define how long to try to connect to the local
# domain socket when setting up a persistent connection. The default value is
# 1 second.
#connect_interval = 1
# The command timeout value defines the amount of time to wait for a command
# or RPC call before timing out. The value for the command timeout must
# be less than the value of the persistent connection idle timeout (connect_timeout)
# The default value is 10 second.
#command_timeout = 10
[accelerate]
#accelerate_port = 5099

View file

@ -1323,15 +1323,6 @@ PARAMIKO_RECORD_HOST_KEYS:
value_type: boolean
vars: []
yaml: {key: paramiko_connection.record_host_keys}
PERSISTENT_CONNECT_INTERVAL:
default: 1
desc: 'TODO: write it'
env: [{name: ANSIBLE_PERSISTENT_CONNECT_INTERVAL}]
ini:
- {key: connect_interval, section: persistent_connection}
value_type: integer
vars: []
yaml: {key: persistent_connection.connect_interval}
PERSISTENT_CONTROL_PATH_DIR:
default: ~/.ansible/pc
desc: 'TODO: write it'
@ -1340,15 +1331,6 @@ PERSISTENT_CONTROL_PATH_DIR:
- {key: control_path_dir, section: persistent_connection}
vars: []
yaml: {key: persistent_connection.control_path_dir}
PERSISTENT_CONNECT_RETRIES:
default: 30
desc: 'TODO: write it'
env: [{name: ANSIBLE_PERSISTENT_CONNECT_RETRIES}]
ini:
- {key: connect_retries, section: persistent_connection}
value_type: integer
vars: []
yaml: {key: persistent_connection.connect_retries}
PERSISTENT_CONNECT_TIMEOUT:
default: 30
desc: 'TODO: write it'
@ -1358,6 +1340,24 @@ PERSISTENT_CONNECT_TIMEOUT:
value_type: integer
vars: []
yaml: {key: persistent_connection.connect_timeout}
PERSISTENT_CONNECT_RETRY_TIMEOUT:
default: 15
desc: 'TODO: write it'
env: [{name: ANSIBLE_PERSISTENT_CONNECT_RETRY_TIMEOUT}]
ini:
- {key: connect_retry_timeout, section: persistent_connection}
value_type: integer
vars: []
yaml: {key: persistent_connection.connect_retry_timeout}
PERSISTENT_COMMAND_TIMEOUT:
default: 10
desc: 'TODO: write it'
env: [{name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT}]
ini:
- {key: command_timeout, section: persistent_connection}
value_type: integer
vars: []
yaml: {key: persistent_connection.command_timeout}
RETRY_FILES_ENABLED:
default: True
desc: This controls whether a failed Ansible playbook should create a .retry file.

View file

@ -43,9 +43,7 @@ aireos_argument_spec = {
}
# Add argument's default value here
ARGS_DEFAULT_VALUE = {
'timeout': 10
}
ARGS_DEFAULT_VALUE = {}
def sanitize(resp):

View file

@ -38,14 +38,12 @@ aruba_argument_spec = {
'username': dict(fallback=(env_fallback, ['ANSIBLE_NET_USERNAME'])),
'password': dict(fallback=(env_fallback, ['ANSIBLE_NET_PASSWORD']), no_log=True),
'ssh_keyfile': dict(fallback=(env_fallback, ['ANSIBLE_NET_SSH_KEYFILE']), type='path'),
'timeout': dict(type='int', default=10),
'timeout': dict(type='int'),
'provider': dict(type='dict')
}
# Add argument's default value here
ARGS_DEFAULT_VALUE = {
'timeout': 10
}
ARGS_DEFAULT_VALUE = {}
def get_argspec():

View file

@ -50,9 +50,7 @@ junos_argument_spec = {
}
# Add argument's default value here
ARGS_DEFAULT_VALUE = {
'timeout': 10
}
ARGS_DEFAULT_VALUE = {}
def get_argspec():

View file

@ -57,8 +57,7 @@ nxos_argument_spec = {
# Add argument's default value here
ARGS_DEFAULT_VALUE = {
'transport': 'cli',
'timeout': 10
'transport': 'cli'
}

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.aireos import aireos_argument_spec
@ -54,7 +55,7 @@ class ActionModule(_ActionModule):
pc.port = provider['port'] or self._play_context.port or 22
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.aruba import aruba_argument_spec
@ -55,7 +56,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)

View file

@ -23,6 +23,7 @@ import sys
import copy
import json
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.asa import asa_argument_spec
@ -57,7 +58,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.six import iteritems
from ansible.module_utils.ce import ce_argument_spec
@ -57,7 +58,7 @@ class ActionModule(_ActionModule):
pc.port = int(provider['port']) or int(self._play_context.port) or 22
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
self._task.args['provider'] = provider.update(
host=pc.remote_addr,
port=pc.port,

View file

@ -24,6 +24,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.six import iteritems
from ansible.module_utils.dellos10 import dellos10_argument_spec
@ -56,7 +57,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -21,6 +21,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.six import iteritems
from ansible.module_utils.dellos6 import dellos6_argument_spec
@ -52,7 +53,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -24,6 +24,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.six import iteritems
from ansible.module_utils.dellos9 import dellos9_argument_spec
@ -56,7 +57,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.eos import ARGS_DEFAULT_VALUE, eos_argument_spec
from ansible.module_utils.six import iteritems
@ -58,7 +59,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.ios import ios_argument_spec
@ -55,7 +56,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
pc.become = provider['authorize'] or False
pc.become_pass = provider['auth_pass']

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.iosxr import iosxr_argument_spec
from ansible.module_utils.six import iteritems
@ -54,7 +55,7 @@ class ActionModule(_ActionModule):
pc.port = provider['port'] or self._play_context.port or 22
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.junos import junos_argument_spec
from ansible.module_utils.six import iteritems
@ -68,7 +69,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)

View file

@ -20,6 +20,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action import ActionBase
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.six import iteritems
@ -59,7 +60,7 @@ class ActionModule(ActionBase):
play_context.remote_user = self.provider['username'] or self._play_context.connection_user
play_context.password = self.provider['password'] or self._play_context.password
play_context.private_key_file = self.provider['ssh_keyfile'] or self._play_context.private_key_file
play_context.timeout = self.provider['timeout'] or self._play_context.timeout
play_context.timeout = self.provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
if 'authorize' in self.provider.keys():
play_context.become = self.provider['authorize'] or False
play_context.become_pass = self.provider['auth_pass']

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.nxos import nxos_argument_spec
@ -58,7 +59,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
self._task.args['provider'] = provider.update(
host=pc.remote_addr,
port=pc.port,

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.sros import sros_argument_spec
from ansible.module_utils.basic import AnsibleFallbackNotFound
@ -55,7 +56,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)

View file

@ -22,6 +22,7 @@ __metaclass__ = type
import sys
import copy
from ansible import constants as C
from ansible.plugins.action.normal import ActionModule as _ActionModule
from ansible.module_utils.basic import AnsibleFallbackNotFound
from ansible.module_utils.six import iteritems
@ -54,7 +55,7 @@ class ActionModule(_ActionModule):
pc.remote_user = provider['username'] or self._play_context.connection_user
pc.password = provider['password'] or self._play_context.password
pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
pc.timeout = provider['timeout'] or self._play_context.timeout
pc.timeout = provider['timeout'] or C.PERSISTENT_COMMAND_TIMEOUT
display.vvv('using connection plugin %s' % pc.connection, pc.remote_addr)
connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin)