cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Catch and handle quickly shutdown connections in reboot plugin (#48389)

Browse source
This commit is contained in:
Sam Doran 2018-11-12 11:30:46 -05:00 committed by GitHub
parent 1465f70eb5
commit 886433bd5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 17 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
lib/ansible/plugins/action/reboot.py
View file

@ -10,7 +10,7 @@ import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from ansible.errors import AnsibleError from ansible.errors import AnsibleError, AnsibleConnectionFailure
from ansible.module_utils._text import to_native, to_text from ansible.module_utils._text import to_native, to_text
from ansible.plugins.action import ActionBase from ansible.plugins.action import ActionBase
@ -61,7 +61,7 @@ class ActionModule(ActionBase):
'linux': '-r {delay_min} "{message}"', 'linux': '-r {delay_min} "{message}"',
'freebsd': '-r +{delay_sec}s "{message}"', 'freebsd': '-r +{delay_sec}s "{message}"',
'sunos': '-y -g {delay_sec} -r "{message}"', 'sunos': '-y -g {delay_sec} -r "{message}"',
'darwin': '-r +{delay_min_macos} "{message}"', 'darwin': '-r +{delay_min} "{message}"',
'openbsd': '-r +{delay_min} "{message}"', 'openbsd': '-r +{delay_min} "{message}"',
} }
@ -88,16 +88,11 @@ class ActionModule(ActionBase):
if pre_reboot_delay < 0: if pre_reboot_delay < 0:
pre_reboot_delay = 0 pre_reboot_delay = 0
# Convert seconds to minutes for Linux. If less that 60, set it to 0 except for macOS which will # Convert seconds to minutes. If less that 60, set it to 0.
# sever the connection too quickly if set to 0, so set that to 1.
# We could simplify this by setting them both to 1, but I think of all the time that
# people will lose waiting for that extra 1 minute delay and want to give them their
# lives back.
delay_min = pre_reboot_delay // 60 delay_min = pre_reboot_delay // 60
delay_min_macos = delay_min | 1
msg = self._task.args.get('msg', self.DEFAULT_REBOOT_MESSAGE) msg = self._task.args.get('msg', self.DEFAULT_REBOOT_MESSAGE)
shutdown_command_args = shutdown_command_args.format(delay_sec=pre_reboot_delay, delay_min=delay_min, delay_min_macos=delay_min_macos, message=msg) shutdown_command_args = shutdown_command_args.format(delay_sec=pre_reboot_delay, delay_min=delay_min, message=msg)
reboot_command = '%s %s' % (shutdown_command, shutdown_command_args) reboot_command = '%s %s' % (shutdown_command, shutdown_command_args)
return reboot_command return reboot_command
@ -187,7 +182,8 @@ class ActionModule(ActionBase):
fail_sleep = max_fail_sleep + random_int fail_sleep = max_fail_sleep + random_int
if action_desc: if action_desc:
display.debug("{0}: {1} fail '{2}', retrying in {3:.4} seconds...".format(self._task.action, action_desc, to_text(e), fail_sleep)) display.debug("{0}: {1} fail '{2}', retrying in {3:.4} seconds...".format(self._task.action, action_desc,
to_text(e).splitlines()[-1], fail_sleep))
fail_count += 1 fail_count += 1
time.sleep(fail_sleep) time.sleep(fail_sleep)
@ -197,8 +193,17 @@ class ActionModule(ActionBase):
display.debug("%s: rebooting server" % self._task.action) display.debug("%s: rebooting server" % self._task.action)
remote_command = self.construct_command() remote_command = self.construct_command()
reboot_result = self._low_level_execute_command(remote_command, sudoable=self.DEFAULT_SUDOABLE)
result = {} result = {}
reboot_result = {}
try:
reboot_result = self._low_level_execute_command(remote_command, sudoable=self.DEFAULT_SUDOABLE)
except AnsibleConnectionFailure as e:
# If the connection is closed too quickly due to the system being shutdown, carry on
display.debug('%s: AnsibleConnectionFailure caught and handled: %s' % (self._task.action, to_native(e)))
reboot_result['rc'] = 0
result['start'] = datetime.utcnow() result['start'] = datetime.utcnow()
if reboot_result['rc'] != 0: if reboot_result['rc'] != 0:
@ -230,11 +235,11 @@ class ActionModule(ActionBase):
self.do_until_success_or_timeout(self.check_boot_time, reboot_timeout, action_desc="boot_time check") self.do_until_success_or_timeout(self.check_boot_time, reboot_timeout, action_desc="boot_time check")
if connect_timeout: if connect_timeout:
# reset the connection to clear the custom connection timeout
try: try:
self._connection.set_option("connection_timeout", connect_timeout) self._connection.set_option("connection_timeout", connect_timeout)
self._connection.reset() self._connection.reset()
except (AnsibleError, AttributeError) as e: except (AnsibleError, AttributeError) as e:
# reset the connection to clear the custom connection timeout
display.debug("Failed to reset connection_timeout back to default: %s" % to_text(e)) display.debug("Failed to reset connection_timeout back to default: %s" % to_text(e))
# finally run test command to ensure everything is working # finally run test command to ensure everything is working