cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[cloud] Allow S3 module to correctly delete objects by groups of <= 1000 (#28068)

Browse source
This commit is contained in:
Sloane Hertel 2017-08-11 15:30:50 -04:00 committed by Ryan Brown
parent d89dd75769
commit 3cb1c38ecc
1 changed files with 5 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
lib/ansible/modules/cloud/amazon/s3.py
View file

@ -327,8 +327,7 @@ def create_bucket(module, s3, bucket, location=None):
def paginated_list(s3, **pagination_params): def paginated_list(s3, **pagination_params):
pg = s3.get_paginator('list_objects_v2') pg = s3.get_paginator('list_objects_v2')
for page in pg.paginate(**pagination_params): for page in pg.paginate(**pagination_params):
for data in page.get('Contents', {}): yield [data['Key'] for data in page.get('Contents', [])]
yield data['Key']
def list_keys(module, s3, bucket, prefix, marker, max_keys): def list_keys(module, s3, bucket, prefix, marker, max_keys):
@ -336,7 +335,7 @@ def list_keys(module, s3, bucket, prefix, marker, max_keys):
for param_name, param_value in (('Prefix', prefix), ('StartAfter', marker), ('MaxKeys', max_keys)): for param_name, param_value in (('Prefix', prefix), ('StartAfter', marker), ('MaxKeys', max_keys)):
pagination_params[param_name] = param_value pagination_params[param_name] = param_value
try: try:
keys = [key for key in paginated_list(s3, **pagination_params)] keys = sum(paginated_list(s3, **pagination_params), [])
module.exit_json(msg="LIST operation complete", s3_keys=keys) module.exit_json(msg="LIST operation complete", s3_keys=keys)
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e:
module.fail_json(msg="Failed while listing the keys in the bucket {0}".format(bucket), module.fail_json(msg="Failed while listing the keys in the bucket {0}".format(bucket),
@ -352,9 +351,9 @@ def delete_bucket(module, s3, bucket):
if exists is False: if exists is False:
return False return False
# if there are contents then we need to delete them before we can delete the bucket # if there are contents then we need to delete them before we can delete the bucket
keys = [{'Key': key} for key in paginated_list(s3, Bucket=bucket)] for keys in paginated_list(s3, Bucket=bucket):
if keys: formatted_keys = [{'Key': key} for key in keys]
s3.delete_objects(Bucket=bucket, Delete={'Objects': keys}) s3.delete_objects(Bucket=bucket, Delete={'Objects': formatted_keys})
s3.delete_bucket(Bucket=bucket) s3.delete_bucket(Bucket=bucket)
return True return True
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e: