cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[cloud] Make ec2_asg more resilient using AWSRetry around boto3 calls (#27598)

Browse source 
* Add AWSRetry to ec2_asg

* Paginate describing ASGs and launch configurations

pass connection to delete_asg

Fix a couple little bugs

* Use boto3's pagination build_full_result()
This commit is contained in:
Sloane Hertel 2017-08-11 11:30:23 -04:00 committed by Ryan Brown
parent e4cd899363
commit 40eb349ac6
1 changed files with 149 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

234
lib/ansible/modules/cloud/amazon/ec2_asg.py
View file

@ -394,6 +394,106 @@ ASG_ATTRIBUTES = ('AvailabilityZones', 'DefaultCooldown', 'DesiredCapacity',
INSTANCE_ATTRIBUTES = ('instance_id', 'health_status', 'lifecycle_state', 'launch_config_name') INSTANCE_ATTRIBUTES = ('instance_id', 'health_status', 'lifecycle_state', 'launch_config_name')
backoff_params = dict(tries=10, delay=3, backoff=1.5)
@AWSRetry.backoff(**backoff_params)
def describe_autoscaling_groups(connection, group_name):
pg = connection.get_paginator('describe_auto_scaling_groups')
return pg.paginate(AutoScalingGroupNames=[group_name]).build_full_result().get('AutoScalingGroups', [])
@AWSRetry.backoff(**backoff_params)
def deregister_lb_instances(connection, lb_name, instance_id):
connection.deregister_instances_from_load_balancer(LoadBalancerName=lb_name, Instances=[dict(InstanceId=instance_id)])
@AWSRetry.backoff(**backoff_params)
def describe_instance_health(connection, lb_name, instances):
params = dict(LoadBalancerName=lb_name)
if instances:
params.update(Instances=instances)
return connection.describe_instance_health(**params)
@AWSRetry.backoff(**backoff_params)
def describe_target_health(connection, target_group_arn, instances):
return connection.describe_target_health(TargetGroupArn=target_group_arn, Targets=instances)
@AWSRetry.backoff(**backoff_params)
def suspend_asg_processes(connection, asg_name, processes):
connection.suspend_processes(AutoScalingGroupName=asg_name, ScalingProcesses=processes)
@AWSRetry.backoff(**backoff_params)
def resume_asg_processes(connection, asg_name, processes):
connection.resume_processes(AutoScalingGroupName=asg_name, ScalingProcesses=processes)
@AWSRetry.backoff(**backoff_params)
def describe_launch_configurations(connection, launch_config_name):
pg = connection.get_paginator('describe_launch_configurations')
return pg.paginate(LaunchConfigurationNames=[launch_config_name]).build_full_result()
@AWSRetry.backoff(**backoff_params)
def create_asg(connection, **params):
connection.create_auto_scaling_group(**params)
@AWSRetry.backoff(**backoff_params)
def put_notification_config(connection, asg_name, topic_arn, notification_types):
connection.put_notification_configuration(
AutoScalingGroupName=asg_name,
TopicARN=topic_arn,
NotificationTypes=notification_types
)
@AWSRetry.backoff(**backoff_params)
def del_notification_config(connection, asg_name, topic_arn):
connection.delete_notification_configuration(
AutoScalingGroupName=asg_name,
TopicARN=topic_arn
)
@AWSRetry.backoff(**backoff_params)
def attach_load_balancers(connection, asg_name, load_balancers):
connection.attach_load_balancers(AutoScalingGroupName=asg_name, LoadBalancerNames=load_balancers)
@AWSRetry.backoff(**backoff_params)
def detach_load_balancers(connection, asg_name, load_balancers):
connection.detach_load_balancers(AutoScalingGroupName=asg_name, LoadBalancerNames=load_balancers)
@AWSRetry.backoff(**backoff_params)
def attach_lb_target_groups(connection, asg_name, target_group_arns):
connection.attach_load_balancer_target_groups(AutoScalingGroupName=asg_name, TargetGroupARNs=target_group_arns)
@AWSRetry.backoff(**backoff_params)
def detach_lb_target_groups(connection, asg_name, target_group_arns):
connection.detach_load_balancer_target_groups(AutoScalingGroupName=asg_name, TargetGroupARNs=target_group_arns)
@AWSRetry.backoff(**backoff_params)
def update_asg(connection, **params):
connection.update_auto_scaling_group(**params)
@AWSRetry.backoff(**backoff_params)
def delete_asg(connection, asg_name, force_delete):
connection.delete_auto_scaling_group(AutoScalingGroupName=asg_name, ForceDelete=force_delete)
@AWSRetry.backoff(**backoff_params)
def terminate_asg_instance(connection, instance_id, decrement_capacity):
connection.terminate_instance_in_auto_scaling_group(InstanceId=instance_id,
ShouldDecrementDesiredCapacity=decrement_capacity)
def enforce_required_arguments(module): def enforce_required_arguments(module):
''' As many arguments are not required for autoscale group deletion ''' As many arguments are not required for autoscale group deletion
@ -471,7 +571,7 @@ def get_properties(autoscaling_group, module):
def elb_dreg(asg_connection, module, group_name, instance_id): def elb_dreg(asg_connection, module, group_name, instance_id):
region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True)
as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(asg_connection, group_name)[0]
wait_timeout = module.params.get('wait_timeout') wait_timeout = module.params.get('wait_timeout')
count = 1 count = 1
if as_group['LoadBalancerNames'] and as_group['HealthCheckType'] == 'ELB': if as_group['LoadBalancerNames'] and as_group['HealthCheckType'] == 'ELB':
@ -485,15 +585,14 @@ def elb_dreg(asg_connection, module, group_name, instance_id):
return return
for lb in as_group['LoadBalancerNames']: for lb in as_group['LoadBalancerNames']:
elb_connection.deregister_instances_from_load_balancer(LoadBalancerName=lb, deregister_lb_instances(elb_connection, lb, instance_id)
Instances=[dict(InstanceId=instance_id)])
log.debug("De-registering {0} from ELB {1}".format(instance_id, lb)) log.debug("De-registering {0} from ELB {1}".format(instance_id, lb))
wait_timeout = time.time() + wait_timeout wait_timeout = time.time() + wait_timeout
while wait_timeout > time.time() and count > 0: while wait_timeout > time.time() and count > 0:
count = 0 count = 0
for lb in as_group['LoadBalancerNames']: for lb in as_group['LoadBalancerNames']:
lb_instances = elb_connection.describe_instance_health(LoadBalancerName=lb) lb_instances = describe_instance_health(elb_connection, lb, [])
for i in lb_instances['InstanceStates']: for i in lb_instances['InstanceStates']:
if i['InstanceId'] == instance_id and i['State'] == "InService": if i['InstanceId'] == instance_id and i['State'] == "InService":
count += 1 count += 1
@ -507,7 +606,7 @@ def elb_dreg(asg_connection, module, group_name, instance_id):
def elb_healthy(asg_connection, elb_connection, module, group_name): def elb_healthy(asg_connection, elb_connection, module, group_name):
healthy_instances = set() healthy_instances = set()
as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(asg_connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
# get healthy, inservice instances from ASG # get healthy, inservice instances from ASG
instances = [] instances = []
@ -521,7 +620,7 @@ def elb_healthy(asg_connection, elb_connection, module, group_name):
# we catch a race condition that sometimes happens if the instance exists in the ASG # we catch a race condition that sometimes happens if the instance exists in the ASG
# but has not yet show up in the ELB # but has not yet show up in the ELB
try: try:
lb_instances = elb_connection.describe_instance_health(LoadBalancerName=lb, Instances=instances) lb_instances = describe_instance_health(elb_connection, lb, instances)
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'InvalidInstance': if e.response['Error']['Code'] == 'InvalidInstance':
return None return None
@ -541,7 +640,7 @@ def elb_healthy(asg_connection, elb_connection, module, group_name):
def tg_healthy(asg_connection, elbv2_connection, module, group_name): def tg_healthy(asg_connection, elbv2_connection, module, group_name):
healthy_instances = set() healthy_instances = set()
as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(asg_connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
# get healthy, inservice instances from ASG # get healthy, inservice instances from ASG
instances = [] instances = []
@ -555,7 +654,7 @@ def tg_healthy(asg_connection, elbv2_connection, module, group_name):
# we catch a race condition that sometimes happens if the instance exists in the ASG # we catch a race condition that sometimes happens if the instance exists in the ASG
# but has not yet show up in the ELB # but has not yet show up in the ELB
try: try:
tg_instances = elbv2_connection.describe_target_health(TargetGroupArn=tg, Targets=instances) tg_instances = describe_target_health(elbv2_connection, tg, instances)
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'InvalidInstance': if e.response['Error']['Code'] == 'InvalidInstance':
return None return None
@ -579,7 +678,7 @@ def wait_for_elb(asg_connection, module, group_name):
# if the health_check_type is ELB, we want to query the ELBs directly for instance # if the health_check_type is ELB, we want to query the ELBs directly for instance
# status as to avoid health_check_grace period that is awarded to ASG instances # status as to avoid health_check_grace period that is awarded to ASG instances
as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(asg_connection, group_name)[0]
if as_group.get('LoadBalancerNames') and as_group.get('HealthCheckType') == 'ELB': if as_group.get('LoadBalancerNames') and as_group.get('HealthCheckType') == 'ELB':
log.debug("Waiting for ELB to consider instances healthy.") log.debug("Waiting for ELB to consider instances healthy.")
@ -609,7 +708,7 @@ def wait_for_target_group(asg_connection, module, group_name):
# if the health_check_type is ELB, we want to query the ELBs directly for instance # if the health_check_type is ELB, we want to query the ELBs directly for instance
# status as to avoid health_check_grace period that is awarded to ASG instances # status as to avoid health_check_grace period that is awarded to ASG instances
as_group = asg_connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(asg_connection, group_name)[0]
if as_group.get('TargetGroupARNs') and as_group.get('HealthCheckType') == 'ELB': if as_group.get('TargetGroupARNs') and as_group.get('HealthCheckType') == 'ELB':
log.debug("Waiting for Target Group to consider instances healthy.") log.debug("Waiting for Target Group to consider instances healthy.")
@ -647,10 +746,10 @@ def suspend_processes(ec2_connection, as_group, module):
resume_processes = list(suspended_processes - suspend_processes) resume_processes = list(suspended_processes - suspend_processes)
if resume_processes: if resume_processes:
ec2_connection.resume_processes(AutoScalingGroupName=module.params.get('name'), ScalingProcesses=resume_processes) resume_asg_processes(ec2_connection, module.params.get('name'), resume_processes)
if suspend_processes: if suspend_processes:
ec2_connection.suspend_processes(AutoScalingGroupName=module.params.get('name'), ScalingProcesses=list(suspend_processes)) suspend_asg_processes(ec2_connection, module.params.get('name'), list(suspend_processes))
return True return True
@ -703,7 +802,7 @@ def create_autoscaling_group(connection, module):
availability_zones = module.params['availability_zones'] = [zone['ZoneName'] for availability_zones = module.params['availability_zones'] = [zone['ZoneName'] for
zone in ec2_connection.describe_availability_zones()['AvailabilityZones']] zone in ec2_connection.describe_availability_zones()['AvailabilityZones']]
enforce_required_arguments(module) enforce_required_arguments(module)
launch_configs = connection.describe_launch_configurations(LaunchConfigurationNames=[launch_config_name]) launch_configs = describe_launch_configurations(connection, launch_config_name)
if len(launch_configs['LaunchConfigurations']) == 0: if len(launch_configs['LaunchConfigurations']) == 0:
module.fail_json(msg="No launch config found with name %s" % launch_config_name) module.fail_json(msg="No launch config found with name %s" % launch_config_name)
ag = dict( ag = dict(
@ -729,9 +828,9 @@ def create_autoscaling_group(connection, module):
ag['TargetGroupARNs'] = target_group_arns ag['TargetGroupARNs'] = target_group_arns
try: try:
connection.create_auto_scaling_group(**ag) create_asg(connection, **ag)
all_ag = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'] all_ag = describe_autoscaling_groups(connection, group_name)
if len(all_ag) == 0: if len(all_ag) == 0:
module.fail_json(msg="No auto scaling group found with the name %s" % group_name) module.fail_json(msg="No auto scaling group found with the name %s" % group_name)
as_group = all_ag[0] as_group = all_ag[0]
@ -744,12 +843,8 @@ def create_autoscaling_group(connection, module):
if target_group_arns: if target_group_arns:
wait_for_target_group(connection, module, group_name) wait_for_target_group(connection, module, group_name)
if notification_topic: if notification_topic:
connection.put_notification_configuration( put_notification_config(connection, group_name, notification_topic, notification_types)
AutoScalingGroupName=group_name, as_group = describe_autoscaling_groups(connection, group_name)[0]
TopicARN=notification_topic,
NotificationTypes=notification_types
)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0]
asg_properties = get_properties(as_group, module) asg_properties = get_properties(as_group, module)
changed = True changed = True
return changed, asg_properties return changed, asg_properties
@ -790,10 +885,7 @@ def create_autoscaling_group(connection, module):
if load_balancers and not as_group['LoadBalancerNames']: if load_balancers and not as_group['LoadBalancerNames']:
changed = True changed = True
try: try:
connection.attach_load_balancers( attach_load_balancers(connection, group_name, load_balancers)
AutoScalingGroupName=group_name,
LoadBalancerNames=load_balancers
)
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
module.fail_json(msg="Failed to update Autoscaling Group.", module.fail_json(msg="Failed to update Autoscaling Group.",
exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
@ -813,29 +905,20 @@ def create_autoscaling_group(connection, module):
elbs_to_detach = has_elbs.difference(wanted_elbs) elbs_to_detach = has_elbs.difference(wanted_elbs)
if elbs_to_detach: if elbs_to_detach:
changed = True changed = True
connection.detach_load_balancers( detach_load_balancers(connection, group_name, list(elbs_to_detach))
AutoScalingGroupName=group_name,
LoadBalancerNames=list(elbs_to_detach)
)
if wanted_elbs - has_elbs: if wanted_elbs - has_elbs:
# if has contains less than wanted, then we need to add some # if has contains less than wanted, then we need to add some
elbs_to_attach = wanted_elbs.difference(has_elbs) elbs_to_attach = wanted_elbs.difference(has_elbs)
if elbs_to_attach: if elbs_to_attach:
changed = True changed = True
connection.attach_load_balancers( attach_load_balancers(connection, group_name, list(elbs_to_attach))
AutoScalingGroupName=group_name,
LoadBalancerNames=list(elbs_to_attach)
)
# Handle target group attachments/detachments # Handle target group attachments/detachments
# Attach target groups if they are specified but none currently exist # Attach target groups if they are specified but none currently exist
if target_group_arns and not as_group['TargetGroupARNs']: if target_group_arns and not as_group['TargetGroupARNs']:
changed = True changed = True
try: try:
connection.attach_load_balancer_target_groups( attach_lb_target_groups(connection, group_name, target_group_arns)
AutoScalingGroupName=group_name,
TargetGroupARNs=target_group_arns
)
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
module.fail_json(msg="Failed to update Autoscaling Group.", module.fail_json(msg="Failed to update Autoscaling Group.",
exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
@ -850,19 +933,13 @@ def create_autoscaling_group(connection, module):
tgs_to_detach = has_tgs.difference(wanted_tgs) tgs_to_detach = has_tgs.difference(wanted_tgs)
if tgs_to_detach: if tgs_to_detach:
changed = True changed = True
connection.detach_load_balancer_target_groups( detach_lb_target_groups(connection, group_name, tgs_to_detach)
AutoScalingGroupName=group_name,
TargetGroupARNs=list(tgs_to_detach)
)
if wanted_tgs.issuperset(has_tgs): if wanted_tgs.issuperset(has_tgs):
# if has contains less than wanted, then we need to add some # if has contains less than wanted, then we need to add some
tgs_to_attach = wanted_tgs.difference(has_tgs) tgs_to_attach = wanted_tgs.difference(has_tgs)
if tgs_to_attach: if tgs_to_attach:
changed = True changed = True
connection.attach_load_balancer_target_groups( attach_lb_target_groups(connection, group_name, tgs_to_attach)
AutoScalingGroupName=group_name,
TargetGroupARNs=list(tgs_to_attach)
)
# check for attributes that aren't required for updating an existing ASG # check for attributes that aren't required for updating an existing ASG
desired_capacity = desired_capacity or as_group['DesiredCapacity'] desired_capacity = desired_capacity or as_group['DesiredCapacity']
@ -870,7 +947,7 @@ def create_autoscaling_group(connection, module):
max_size = max_size or as_group['MaxSize'] max_size = max_size or as_group['MaxSize']
launch_config_name = launch_config_name or as_group['LaunchConfigurationName'] launch_config_name = launch_config_name or as_group['LaunchConfigurationName']
launch_configs = connection.describe_launch_configurations(LaunchConfigurationNames=[launch_config_name]) launch_configs = describe_launch_configurations(connection, launch_config_name)
if len(launch_configs['LaunchConfigurations']) == 0: if len(launch_configs['LaunchConfigurations']) == 0:
module.fail_json(msg="No launch config found with name %s" % launch_config_name) module.fail_json(msg="No launch config found with name %s" % launch_config_name)
ag = dict( ag = dict(
@ -887,15 +964,11 @@ def create_autoscaling_group(connection, module):
ag['AvailabilityZones'] = availability_zones ag['AvailabilityZones'] = availability_zones
if vpc_zone_identifier: if vpc_zone_identifier:
ag['VPCZoneIdentifier'] = vpc_zone_identifier ag['VPCZoneIdentifier'] = vpc_zone_identifier
connection.update_auto_scaling_group(**ag) update_asg(connection, **ag)
if notification_topic: if notification_topic:
try: try:
connection.put_notification_configuration( put_notification_config(connection, group_name, notification_topic, notification_types)
AutoScalingGroupName=group_name,
TopicARN=notification_topic,
NotificationTypes=notification_types
)
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
module.fail_json(msg="Failed to update Autoscaling Group notifications.", module.fail_json(msg="Failed to update Autoscaling Group notifications.",
exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response))
@ -912,8 +985,7 @@ def create_autoscaling_group(connection, module):
wait_for_target_group(connection, module, group_name) wait_for_target_group(connection, module, group_name)
try: try:
as_group = connection.describe_auto_scaling_groups( as_group = describe_autoscaling_groups(connection, group_name)[0]
AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0]
asg_properties = get_properties(as_group, module) asg_properties = get_properties(as_group, module)
if asg_properties != initial_asg_properties: if asg_properties != initial_asg_properties:
changed = True changed = True
@ -930,26 +1002,19 @@ def delete_autoscaling_group(connection, module):
wait_timeout = module.params.get('wait_timeout') wait_timeout = module.params.get('wait_timeout')
if notification_topic: if notification_topic:
connection.delete_notification_configuration( del_notification_config(connection, group_name, notification_topic)
AutoScalingGroupName=group_name, groups = describe_autoscaling_groups(connection, group_name)
TopicARN=notification_topic
)
describe_response = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])
groups = describe_response.get('AutoScalingGroups')
if groups: if groups:
if not wait_for_instances: if not wait_for_instances:
connection.delete_auto_scaling_group(AutoScalingGroupName=group_name, ForceDelete=True) delete_asg(connection, group_name, force_delete=True)
return True return True
wait_timeout = time.time() + wait_timeout wait_timeout = time.time() + wait_timeout
connection.update_auto_scaling_group( updated_params = dict(AutoScalingGroupName=group_name, MinSize=0, MaxSize=0, DesiredCapacity=0)
AutoScalingGroupName=group_name, update_asg(connection, **updated_params)
MinSize=0, MaxSize=0,
DesiredCapacity=0)
instances = True instances = True
while instances and wait_for_instances and wait_timeout >= time.time(): while instances and wait_for_instances and wait_timeout >= time.time():
tmp_groups = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name]).get( tmp_groups = describe_autoscaling_groups(connection, group_name)
'AutoScalingGroups')
if tmp_groups: if tmp_groups:
tmp_group = tmp_groups[0] tmp_group = tmp_groups[0]
if not tmp_group.get('Instances'): if not tmp_group.get('Instances'):
@ -960,8 +1025,8 @@ def delete_autoscaling_group(connection, module):
# waiting took too long # waiting took too long
module.fail_json(msg="Waited too long for old instances to terminate. %s" % time.asctime()) module.fail_json(msg="Waited too long for old instances to terminate. %s" % time.asctime())
connection.delete_auto_scaling_group(AutoScalingGroupName=group_name) delete_asg(connection, group_name, force_delete=False)
while len(connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name]).get('AutoScalingGroups')): while describe_autoscaling_groups(connection, group_name):
time.sleep(5) time.sleep(5)
return True return True
@ -982,7 +1047,7 @@ def update_size(connection, group, max_size, min_size, dc):
updated_group['MinSize'] = min_size updated_group['MinSize'] = min_size
updated_group['MaxSize'] = max_size updated_group['MaxSize'] = max_size
updated_group['DesiredCapacity'] = dc updated_group['DesiredCapacity'] = dc
connection.update_auto_scaling_group(**updated_group) update_asg(connection, **updated_group)
def replace(connection, module): def replace(connection, module):
@ -995,7 +1060,7 @@ def replace(connection, module):
lc_check = module.params.get('lc_check') lc_check = module.params.get('lc_check')
replace_instances = module.params.get('replace_instances') replace_instances = module.params.get('replace_instances')
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
wait_for_new_inst(module, connection, group_name, wait_timeout, as_group['MinSize'], 'viable_instances') wait_for_new_inst(module, connection, group_name, wait_timeout, as_group['MinSize'], 'viable_instances')
props = get_properties(as_group, module) props = get_properties(as_group, module)
instances = props['instances'] instances = props['instances']
@ -1010,7 +1075,7 @@ def replace(connection, module):
if num_new_inst_needed == 0 and old_instances: if num_new_inst_needed == 0 and old_instances:
log.debug("No new instances needed, but old instances are present. Removing old instances") log.debug("No new instances needed, but old instances are present. Removing old instances")
terminate_batch(connection, module, old_instances, instances, True) terminate_batch(connection, module, old_instances, instances, True)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
changed = True changed = True
return(changed, props) return(changed, props)
@ -1034,12 +1099,12 @@ def replace(connection, module):
# set temporary settings and wait for them to be reached # set temporary settings and wait for them to be reached
# This should get overwritten if the number of instances left is less than the batch size. # This should get overwritten if the number of instances left is less than the batch size.
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
update_size(connection, as_group, max_size + batch_size, min_size + batch_size, desired_capacity + batch_size) update_size(connection, as_group, max_size + batch_size, min_size + batch_size, desired_capacity + batch_size)
wait_for_new_inst(module, connection, group_name, wait_timeout, as_group['MinSize'], 'viable_instances') wait_for_new_inst(module, connection, group_name, wait_timeout, as_group['MinSize'], 'viable_instances')
wait_for_elb(connection, module, group_name) wait_for_elb(connection, module, group_name)
wait_for_target_group(connection, module, group_name) wait_for_target_group(connection, module, group_name)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
instances = props['instances'] instances = props['instances']
if replace_instances: if replace_instances:
@ -1052,12 +1117,12 @@ def replace(connection, module):
wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size, 'viable_instances') wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size, 'viable_instances')
wait_for_elb(connection, module, group_name) wait_for_elb(connection, module, group_name)
wait_for_target_group(connection, module, group_name) wait_for_target_group(connection, module, group_name)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
if break_early: if break_early:
log.debug("breaking loop") log.debug("breaking loop")
break break
update_size(connection, as_group, max_size, min_size, desired_capacity) update_size(connection, as_group, max_size, min_size, desired_capacity)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
asg_properties = get_properties(as_group, module) asg_properties = get_properties(as_group, module)
log.debug("Rolling update complete.") log.debug("Rolling update complete.")
changed = True changed = True
@ -1115,7 +1180,7 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le
decrement_capacity = False decrement_capacity = False
break_loop = False break_loop = False
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
desired_size = as_group['MinSize'] desired_size = as_group['MinSize']
@ -1134,8 +1199,8 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le
if num_new_inst_needed == 0: if num_new_inst_needed == 0:
decrement_capacity = True decrement_capacity = True
if as_group['MinSize'] != min_size: if as_group['MinSize'] != min_size:
connection.update_auto_scaling_group(AutoScalingGroupName=as_group['AutoScalingGroupName'], updated_params = dict(AutoScalingGroupName=as_group['AutoScalingGroupName'], MinSize=min_size)
MinSize=min_size) update_asg(connection, **updated_params)
log.debug("Updating minimum size back to original of {0}".format(min_size)) log.debug("Updating minimum size back to original of {0}".format(min_size))
# if are some leftover old instances, but we are already at capacity with new ones # if are some leftover old instances, but we are already at capacity with new ones
# we don't want to decrement capacity # we don't want to decrement capacity
@ -1157,8 +1222,7 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le
for instance_id in instances_to_terminate: for instance_id in instances_to_terminate:
elb_dreg(connection, module, group_name, instance_id) elb_dreg(connection, module, group_name, instance_id)
log.debug("terminating instance: {0}".format(instance_id)) log.debug("terminating instance: {0}".format(instance_id))
connection.terminate_instance_in_auto_scaling_group(InstanceId=instance_id, terminate_asg_instance(connection, instance_id, decrement_capacity)
ShouldDecrementDesiredCapacity=decrement_capacity)
# we wait to make sure the machines we marked as Unhealthy are # we wait to make sure the machines we marked as Unhealthy are
# no longer in the list # no longer in the list
@ -1169,14 +1233,14 @@ def terminate_batch(connection, module, replace_instances, initial_instances, le
def wait_for_term_inst(connection, module, term_instances): def wait_for_term_inst(connection, module, term_instances):
wait_timeout = module.params.get('wait_timeout') wait_timeout = module.params.get('wait_timeout')
group_name = module.params.get('name') group_name = module.params.get('name')
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
count = 1 count = 1
wait_timeout = time.time() + wait_timeout wait_timeout = time.time() + wait_timeout
while wait_timeout > time.time() and count > 0: while wait_timeout > time.time() and count > 0:
log.debug("waiting for instances to terminate") log.debug("waiting for instances to terminate")
count = 0 count = 0
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
instance_facts = props['instance_facts'] instance_facts = props['instance_facts']
instances = (i for i in instance_facts if i in term_instances) instances = (i for i in instance_facts if i in term_instances)
@ -1196,7 +1260,7 @@ def wait_for_term_inst(connection, module, term_instances):
def wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size, prop): def wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size, prop):
# make sure we have the latest stats after that last loop. # make sure we have the latest stats after that last loop.
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop])) log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop]))
# now we make sure that we have enough instances in a viable state # now we make sure that we have enough instances in a viable state
@ -1204,7 +1268,7 @@ def wait_for_new_inst(module, connection, group_name, wait_timeout, desired_size
while wait_timeout > time.time() and desired_size > props[prop]: while wait_timeout > time.time() and desired_size > props[prop]:
log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop])) log.debug("Waiting for {0} = {1}, currently {2}".format(prop, desired_size, props[prop]))
time.sleep(10) time.sleep(10)
as_group = connection.describe_auto_scaling_groups(AutoScalingGroupNames=[group_name])['AutoScalingGroups'][0] as_group = describe_autoscaling_groups(connection, group_name)[0]
props = get_properties(as_group, module) props = get_properties(as_group, module)
if wait_timeout <= time.time(): if wait_timeout <= time.time():
# waiting took too long # waiting took too long