From 185a371efbc1665b6be31588bd9eaec6ce5ee870 Mon Sep 17 00:00:00 2001 From: Etherdaemon Date: Wed, 25 Nov 2015 20:36:07 +1000 Subject: [PATCH 1/2] Proposing a wait_for function to ensure elb has been successfully removed --- cloud/amazon/ec2_elb_lb.py | 85 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/cloud/amazon/ec2_elb_lb.py b/cloud/amazon/ec2_elb_lb.py index 1d9b2db283e..71ab931ecd2 100644 --- a/cloud/amazon/ec2_elb_lb.py +++ b/cloud/amazon/ec2_elb_lb.py @@ -107,7 +107,6 @@ options: description: - Wait a specified timeout allowing connections to drain before terminating an instance required: false - default: "None" aliases: [] version_added: "1.8" idle_timeout: @@ -128,6 +127,14 @@ options: - An associative array of stickness policy settings. Policy will be applied to all listeners ( see example ) required: false version_added: "2.0" + wait_for: + description: + - When specified, Ansible will check the status of the load balancer to ensure it has been successfully + removed from AWS. + required: false + default: no + choices: ["yes", "no"] + version_added: 2.0 extends_documentation_fragment: - aws @@ -203,6 +210,13 @@ EXAMPLES = """ name: "test-please-delete" state: absent +# Ensure ELB is gone and wait for check +- local_action: + module: ec2_elb_lb + name: "test-please-delete" + state: absent + wait_for: yes + # Normally, this module will purge any listeners that exist on the ELB # but aren't specified in the listeners parameter. If purge_listeners is # false it leaves them alone @@ -325,7 +339,7 @@ class ElbManager(object): scheme="internet-facing", connection_draining_timeout=None, idle_timeout=None, cross_az_load_balancing=None, access_logs=None, - stickiness=None, region=None, **aws_connect_params): + stickiness=None, wait_for=None, region=None, **aws_connect_params): self.module = module self.name = name @@ -343,6 +357,7 @@ class ElbManager(object): self.cross_az_load_balancing = cross_az_load_balancing self.access_logs = access_logs self.stickiness = stickiness + self.wait_for = wait_for self.aws_connect_params = aws_connect_params self.region = region @@ -351,6 +366,7 @@ class ElbManager(object): self.status = 'gone' self.elb_conn = self._get_elb_connection() self.elb = self._get_elb() + self.ec2_conn = self._get_ec2_connection() def ensure_ok(self): """Create the ELB""" @@ -381,6 +397,14 @@ class ElbManager(object): """Destroy the ELB""" if self.elb: self._delete_elb() + if self.wait_for: + elb_removed = self._wait_for_elb_removed() + # Unfortunately even though the ELB itself is removed quickly + # the interfaces take longer so reliant security groups cannot + # be deleted until the interface has registered as removed. + elb_interface_removed = self._wait_for_elb_interface_removed() + if not (elb_removed and elb_interface_removed): + self.module.fail_json(msg='Timed out waiting for removal of load balancer.') def get_info(self): try: @@ -481,6 +505,50 @@ class ElbManager(object): return info + def _wait_for_elb_removed(self): + polling_increment_secs = 30 + max_retries = 10 + status_achieved = False + + for x in range(0, max_retries): + try: + result = self.elb_conn.get_all_lb_attributes(self.name) + except (boto.exception.BotoServerError, StandardError), e: + if "LoadBalancerNotFound" in e.code: + status_achieved = True + break + else: + time.sleep(polling_increment_secs) + + return status_achieved + + def _wait_for_elb_interface_removed(self): + polling_increment_secs = 30 + max_retries = 10 + status_achieved = False + + elb_interfaces = self.ec2_conn.get_all_network_interfaces( + filters={'attachment.instance-owner-id': 'amazon-elb', + 'description': 'ELB {0}'.format(self.name) }) + + for x in range(0, max_retries): + for interface in elb_interfaces: + try: + result = self.ec2_conn.get_all_network_interfaces(interface.id) + if result == []: + status_achieved = True + break + else: + time.sleep(polling_increment_secs) + except (boto.exception.BotoServerError, StandardError), e: + if 'InvalidNetworkInterfaceID' in e.code: + status_achieved = True + break + else: + self.module.fail_json(msg=str(e)) + + return status_achieved + def _get_elb(self): elbs = self.elb_conn.get_all_load_balancers() for elb in elbs: @@ -495,6 +563,13 @@ class ElbManager(object): except (boto.exception.NoAuthHandlerFound, StandardError), e: self.module.fail_json(msg=str(e)) + def _get_ec2_connection(self): + try: + return connect_to_aws(boto.ec2, self.region, + **self.aws_connect_params) + except (boto.exception.NoAuthHandlerFound, StandardError), e: + self.module.fail_json(msg=str(e)) + def _delete_elb(self): # True if succeeds, exception raised if not result = self.elb_conn.delete_load_balancer(name=self.name) @@ -892,7 +967,8 @@ def main(): idle_timeout={'default': None, 'required': False}, cross_az_load_balancing={'default': None, 'required': False}, stickiness={'default': None, 'required': False, 'type': 'dict'}, - access_logs={'default': None, 'required': False, 'type': 'dict'} + access_logs={'default': None, 'required': False, 'type': 'dict'}, + wait_for={'default': False, 'type': 'bool', 'required': False} ) ) @@ -925,6 +1001,7 @@ def main(): idle_timeout = module.params['idle_timeout'] cross_az_load_balancing = module.params['cross_az_load_balancing'] stickiness = module.params['stickiness'] + wait_for = module.params['wait_for'] if state == 'present' and not listeners: module.fail_json(msg="At least one port is required for ELB creation") @@ -952,7 +1029,7 @@ def main(): subnets, purge_subnets, scheme, connection_draining_timeout, idle_timeout, cross_az_load_balancing, - access_logs, stickiness, + access_logs, stickiness, wait_for, region=region, **aws_connect_params) # check for unsupported attributes for this version of boto From a56ffff51b807c0e4a7780795a95e495ab357a3e Mon Sep 17 00:00:00 2001 From: Etherdaemon Date: Mon, 7 Dec 2015 20:27:31 +1000 Subject: [PATCH 2/2] Update to wait and wait_timeout with a maximum of 10 minutes timeout --- cloud/amazon/ec2_elb_lb.py | 47 +++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/cloud/amazon/ec2_elb_lb.py b/cloud/amazon/ec2_elb_lb.py index 71ab931ecd2..309065bfc2a 100644 --- a/cloud/amazon/ec2_elb_lb.py +++ b/cloud/amazon/ec2_elb_lb.py @@ -127,7 +127,7 @@ options: - An associative array of stickness policy settings. Policy will be applied to all listeners ( see example ) required: false version_added: "2.0" - wait_for: + wait: description: - When specified, Ansible will check the status of the load balancer to ensure it has been successfully removed from AWS. @@ -135,6 +135,13 @@ options: default: no choices: ["yes", "no"] version_added: 2.0 + wait_timeout: + description: + - Used in conjunction with wait. Number of seconds to wait for the elb to be terminated. + A maximum of 600 seconds (10 minutes) is allowed. + required: false + default: 60 + version_added: 2.0 extends_documentation_fragment: - aws @@ -210,12 +217,20 @@ EXAMPLES = """ name: "test-please-delete" state: absent -# Ensure ELB is gone and wait for check +# Ensure ELB is gone and wait for check (for default timeout) - local_action: module: ec2_elb_lb name: "test-please-delete" state: absent - wait_for: yes + wait: yes + +# Ensure ELB is gone and wait for check with timeout value +- local_action: + module: ec2_elb_lb + name: "test-please-delete" + state: absent + wait: yes + wait_timeout: 600 # Normally, this module will purge any listeners that exist on the ELB # but aren't specified in the listeners parameter. If purge_listeners is @@ -339,7 +354,7 @@ class ElbManager(object): scheme="internet-facing", connection_draining_timeout=None, idle_timeout=None, cross_az_load_balancing=None, access_logs=None, - stickiness=None, wait_for=None, region=None, **aws_connect_params): + stickiness=None, wait=None, wait_timeout=None, region=None, **aws_connect_params): self.module = module self.name = name @@ -357,7 +372,8 @@ class ElbManager(object): self.cross_az_load_balancing = cross_az_load_balancing self.access_logs = access_logs self.stickiness = stickiness - self.wait_for = wait_for + self.wait = wait + self.wait_timeout = wait_timeout self.aws_connect_params = aws_connect_params self.region = region @@ -397,7 +413,7 @@ class ElbManager(object): """Destroy the ELB""" if self.elb: self._delete_elb() - if self.wait_for: + if self.wait: elb_removed = self._wait_for_elb_removed() # Unfortunately even though the ELB itself is removed quickly # the interfaces take longer so reliant security groups cannot @@ -506,8 +522,8 @@ class ElbManager(object): return info def _wait_for_elb_removed(self): - polling_increment_secs = 30 - max_retries = 10 + polling_increment_secs = 15 + max_retries = (self.wait_timeout / polling_increment_secs) status_achieved = False for x in range(0, max_retries): @@ -523,8 +539,8 @@ class ElbManager(object): return status_achieved def _wait_for_elb_interface_removed(self): - polling_increment_secs = 30 - max_retries = 10 + polling_increment_secs = 15 + max_retries = (self.wait_timeout / polling_increment_secs) status_achieved = False elb_interfaces = self.ec2_conn.get_all_network_interfaces( @@ -968,7 +984,8 @@ def main(): cross_az_load_balancing={'default': None, 'required': False}, stickiness={'default': None, 'required': False, 'type': 'dict'}, access_logs={'default': None, 'required': False, 'type': 'dict'}, - wait_for={'default': False, 'type': 'bool', 'required': False} + wait={'default': False, 'type': 'bool', 'required': False}, + wait_timeout={'default': 60, 'type': 'int', 'required': False} ) ) @@ -1001,7 +1018,8 @@ def main(): idle_timeout = module.params['idle_timeout'] cross_az_load_balancing = module.params['cross_az_load_balancing'] stickiness = module.params['stickiness'] - wait_for = module.params['wait_for'] + wait = module.params['wait'] + wait_timeout = module.params['wait_timeout'] if state == 'present' and not listeners: module.fail_json(msg="At least one port is required for ELB creation") @@ -1009,6 +1027,9 @@ def main(): if state == 'present' and not (zones or subnets): module.fail_json(msg="At least one availability zone or subnet is required for ELB creation") + if wait_timeout > 600: + module.fail_json(msg='wait_timeout maximum is 600 seconds') + if security_group_names: security_group_ids = [] try: @@ -1029,7 +1050,7 @@ def main(): subnets, purge_subnets, scheme, connection_draining_timeout, idle_timeout, cross_az_load_balancing, - access_logs, stickiness, wait_for, + access_logs, stickiness, wait, wait_timeout, region=region, **aws_connect_params) # check for unsupported attributes for this version of boto