better timeout logic when deleting vm disks. Python 2.4 safe excepts
This commit is contained in:
parent
797a617a1d
commit
c66de51b16
1 changed files with 31 additions and 22 deletions
|
@ -144,6 +144,7 @@ import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from urlparse import urlparse
|
from urlparse import urlparse
|
||||||
|
from ansible.module_utils.facts import * # TimeoutError
|
||||||
|
|
||||||
AZURE_LOCATIONS = ['South Central US',
|
AZURE_LOCATIONS = ['South Central US',
|
||||||
'Central US',
|
'Central US',
|
||||||
|
@ -215,6 +216,23 @@ def _wait_for_completion(azure, promise, wait_timeout, msg):
|
||||||
|
|
||||||
raise WindowsAzureError('Timed out waiting for async operation ' + msg + ' "' + str(promise.request_id) + '" to complete.')
|
raise WindowsAzureError('Timed out waiting for async operation ' + msg + ' "' + str(promise.request_id) + '" to complete.')
|
||||||
|
|
||||||
|
def _delete_disks_when_detached(azure, wait_timeout, disk_names):
|
||||||
|
def _handle_timeout(signum, frame):
|
||||||
|
raise TimeoutError("Timeout reached while waiting for disks to become detached.")
|
||||||
|
|
||||||
|
signal.signal(signal.SIGALRM, _handle_timeout)
|
||||||
|
signal.alarm(wait_timeout)
|
||||||
|
try:
|
||||||
|
while len(disk_names) > 0:
|
||||||
|
for disk_name in disk_names:
|
||||||
|
disk = azure.get_disk(disk_name)
|
||||||
|
if disk.attached_to is None:
|
||||||
|
azure.delete_disk(disk.name, True)
|
||||||
|
disk_names.remove(disk_name)
|
||||||
|
except WindowsAzureError, e:
|
||||||
|
module.fail_json(msg="failed to get or delete disk, error was: %s" % (disk_name, str(e)))
|
||||||
|
finally:
|
||||||
|
signal.alarm(0)
|
||||||
|
|
||||||
def get_ssh_certificate_tokens(module, ssh_cert_path):
|
def get_ssh_certificate_tokens(module, ssh_cert_path):
|
||||||
"""
|
"""
|
||||||
|
@ -268,7 +286,7 @@ def create_virtual_machine(module, azure):
|
||||||
result = azure.create_hosted_service(service_name=name, label=name, location=location)
|
result = azure.create_hosted_service(service_name=name, label=name, location=location)
|
||||||
_wait_for_completion(azure, result, wait_timeout, "create_hosted_service")
|
_wait_for_completion(azure, result, wait_timeout, "create_hosted_service")
|
||||||
changed = True
|
changed = True
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to create the new service, error was: %s" % str(e))
|
module.fail_json(msg="failed to create the new service, error was: %s" % str(e))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -329,13 +347,13 @@ def create_virtual_machine(module, azure):
|
||||||
virtual_network_name=virtual_network_name)
|
virtual_network_name=virtual_network_name)
|
||||||
_wait_for_completion(azure, result, wait_timeout, "create_virtual_machine_deployment")
|
_wait_for_completion(azure, result, wait_timeout, "create_virtual_machine_deployment")
|
||||||
changed = True
|
changed = True
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to create the new virtual machine, error was: %s" % str(e))
|
module.fail_json(msg="failed to create the new virtual machine, error was: %s" % str(e))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
|
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
|
||||||
return (changed, urlparse(deployment.url).hostname, deployment)
|
return (changed, urlparse(deployment.url).hostname, deployment)
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to lookup the deployment information for %s, error was: %s" % (name, str(e)))
|
module.fail_json(msg="failed to lookup the deployment information for %s, error was: %s" % (name, str(e)))
|
||||||
|
|
||||||
|
|
||||||
|
@ -363,9 +381,9 @@ def terminate_virtual_machine(module, azure):
|
||||||
disk_names = []
|
disk_names = []
|
||||||
try:
|
try:
|
||||||
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
|
deployment = azure.get_deployment_by_name(service_name=name, deployment_name=name)
|
||||||
except WindowsAzureMissingResourceError as e:
|
except WindowsAzureMissingResourceError, e:
|
||||||
pass # no such deployment or service
|
pass # no such deployment or service
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to find the deployment, error was: %s" % str(e))
|
module.fail_json(msg="failed to find the deployment, error was: %s" % str(e))
|
||||||
|
|
||||||
# Delete deployment
|
# Delete deployment
|
||||||
|
@ -378,37 +396,28 @@ def terminate_virtual_machine(module, azure):
|
||||||
role_props = azure.get_role(name, deployment.name, role.role_name)
|
role_props = azure.get_role(name, deployment.name, role.role_name)
|
||||||
if role_props.os_virtual_hard_disk.disk_name not in disk_names:
|
if role_props.os_virtual_hard_disk.disk_name not in disk_names:
|
||||||
disk_names.append(role_props.os_virtual_hard_disk.disk_name)
|
disk_names.append(role_props.os_virtual_hard_disk.disk_name)
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to get the role %s, error was: %s" % (role.role_name, str(e)))
|
module.fail_json(msg="failed to get the role %s, error was: %s" % (role.role_name, str(e)))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = azure.delete_deployment(name, deployment.name)
|
result = azure.delete_deployment(name, deployment.name)
|
||||||
_wait_for_completion(azure, result, wait_timeout, "delete_deployment")
|
_wait_for_completion(azure, result, wait_timeout, "delete_deployment")
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to delete the deployment %s, error was: %s" % (deployment.name, str(e)))
|
module.fail_json(msg="failed to delete the deployment %s, error was: %s" % (deployment.name, str(e)))
|
||||||
|
|
||||||
# It's unclear when disks associated with terminated deployment get detatched.
|
# It's unclear when disks associated with terminated deployment get detatched.
|
||||||
# Thus, until the wait_timeout is reached, we continue to delete disks as they
|
# Thus, until the wait_timeout is reached, we continue to delete disks as they
|
||||||
# become detatched by polling the list of remaining disks and examining the state.
|
# become detatched by polling the list of remaining disks and examining the state.
|
||||||
wait_start = time.time()
|
|
||||||
while len(disk_names) > 0:
|
|
||||||
try:
|
try:
|
||||||
for disk_name in disk_names:
|
_delete_disks_when_detached(azure, wait_timeout, disk_names)
|
||||||
disk = azure.get_disk(disk_name)
|
except (WindowsAzureError, TimeoutError), e:
|
||||||
if disk.attached_to is None:
|
module.fail_json(msg=str(e))
|
||||||
azure.delete_disk(disk.name, True)
|
|
||||||
disk_names.remove(disk_name)
|
|
||||||
except WindowsAzureError as e:
|
|
||||||
module.fail_json(msg="failed to get or delete disk, error was: %s" % (disk_name, str(e)))
|
|
||||||
|
|
||||||
if (time.time() - wait_start) > wait_timeout:
|
|
||||||
module.fail_json(msg="Timeout reached while waiting for disks to become detached.")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Now that the vm is deleted, remove the cloud service
|
# Now that the vm is deleted, remove the cloud service
|
||||||
result = azure.delete_hosted_service(service_name=name)
|
result = azure.delete_hosted_service(service_name=name)
|
||||||
_wait_for_completion(azure, result, wait_timeout, "delete_hosted_service")
|
_wait_for_completion(azure, result, wait_timeout, "delete_hosted_service")
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
module.fail_json(msg="failed to delete the service %s, error was: %s" % (name, str(e)))
|
module.fail_json(msg="failed to delete the service %s, error was: %s" % (name, str(e)))
|
||||||
public_dns_name = urlparse(deployment.url).hostname
|
public_dns_name = urlparse(deployment.url).hostname
|
||||||
|
|
||||||
|
@ -510,7 +519,7 @@ class Wrapper(object):
|
||||||
while wait_timeout > time.time():
|
while wait_timeout > time.time():
|
||||||
try:
|
try:
|
||||||
return f()
|
return f()
|
||||||
except WindowsAzureError as e:
|
except WindowsAzureError, e:
|
||||||
if not str(e).lower().find("temporary redirect") == -1:
|
if not str(e).lower().find("temporary redirect") == -1:
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
pass
|
pass
|
||||||
|
|
Loading…
Reference in a new issue