From a87b2e38a033bd77b1594c3a405953cc908c3231 Mon Sep 17 00:00:00 2001 From: "Javier M. Mellid" Date: Wed, 3 Feb 2016 21:22:43 +0100 Subject: [PATCH 1/3] Add Ceph RGW S3 compatibility Ceph Object Gateway (Ceph RGW) is an object storage interface built on top of librados to provide applications with a RESTful gateway to Ceph Storage Clusters: http://docs.ceph.com/docs/master/radosgw/ This patch adds the required bits to handle buckets with the RGW S3 RESTful API properly. It sticks to the AWS behaviour where possible while avoiding not yet implemented features in the Ceph RGW API. Signed-off-by: Javier M. Mellid --- cloud/amazon/s3_bucket.py | 82 ++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/cloud/amazon/s3_bucket.py b/cloud/amazon/s3_bucket.py index ec838d299f1..fa2e4b92e64 100644 --- a/cloud/amazon/s3_bucket.py +++ b/cloud/amazon/s3_bucket.py @@ -16,9 +16,9 @@ DOCUMENTATION = ''' --- module: s3_bucket -short_description: Manage s3 buckets in AWS +short_description: Manage S3 buckets in AWS, Ceph, Walrus and FakeS3 description: - - Manage s3 buckets in AWS + - Manage S3 buckets in AWS, Ceph, Walrus and FakeS3 version_added: "2.0" author: "Rob White (@wimnat)" options: @@ -40,9 +40,13 @@ options: default: null s3_url: description: - - S3 URL endpoint for usage with Eucalypus, fakes3, etc. Otherwise assumes AWS + - S3 URL endpoint for usage with Ceph, Eucalypus, fakes3, etc. Otherwise assumes AWS default: null aliases: [ S3_URL ] + ceph: + description: + - Enable API compatibility with Ceph. It takes into account the S3 API subset working with Ceph in order to provide the same module behaviour where possible. + version_added: "2.2" requester_pays: description: - With Requester Pays buckets, the requester instead of the bucket owner pays the cost of the request and the data download from the bucket. @@ -78,6 +82,12 @@ EXAMPLES = ''' - s3_bucket: name: mys3bucket +# Create a simple s3 bucket on Ceph Rados Gateway +- s3_bucket: + name: mys3bucket + s3_url: http://your-ceph-rados-gateway-server.xxx + ceph: true + # Remove an s3 bucket and any keys it contains - s3_bucket: name: mys3bucket @@ -130,8 +140,8 @@ def create_tags_container(tags): tags_obj.add_tag_set(tag_set) return tags_obj -def create_bucket(connection, module, location): - +def _create_bucket(connection, module, location): + policy = module.params.get("policy") name = module.params.get("name") requester_pays = module.params.get("requester_pays") @@ -258,7 +268,7 @@ def create_bucket(connection, module, location): module.exit_json(changed=changed, name=bucket.name, versioning=versioning_status, requester_pays=requester_pays_status, policy=current_policy, tags=current_tags_dict) -def destroy_bucket(connection, module): +def _destroy_bucket(connection, module): force = module.params.get("force") name = module.params.get("name") @@ -290,6 +300,39 @@ def destroy_bucket(connection, module): module.exit_json(changed=changed) +def _create_bucket_ceph(connection, module, location): + + name = module.params.get("name") + + changed = False + + try: + bucket = connection.get_bucket(name) + except S3ResponseError, e: + try: + bucket = connection.create_bucket(name, location=location) + changed = True + except S3CreateError, e: + module.fail_json(msg=e.message) + + module.exit_json(changed=changed) + +def _destroy_bucket_ceph(connection, module): + + _destroy_bucket(connection, module) + +def create_bucket(connection, module, location, flavour='aws'): + if flavour == 'ceph': + _create_bucket_ceph(connection, module, location) + else: + _create_bucket(connection, module, location) + +def destroy_bucket(connection, module, flavour='aws'): + if flavour == 'ceph': + _destroy_bucket_ceph(connection, module) + else: + _destroy_bucket(connection, module) + def is_fakes3(s3_url): """ Return True if s3_url has scheme fakes3:// """ if s3_url is not None: @@ -319,7 +362,8 @@ def main(): s3_url = dict(aliases=['S3_URL']), state = dict(default='present', choices=['present', 'absent']), tags = dict(required=None, default={}, type='dict'), - versioning = dict(default='no', type='bool') + versioning = dict(default='no', type='bool'), + ceph = dict(default='no', type='bool') ) ) @@ -344,10 +388,27 @@ def main(): if not s3_url and 'S3_URL' in os.environ: s3_url = os.environ['S3_URL'] + ceph = module.params.get('ceph') + + if ceph and not s3_url: + module.fail_json(msg='ceph flavour requires s3_url') + + flavour = 'aws' + # Look at s3_url and tweak connection settings # if connecting to Walrus or fakes3 try: - if is_fakes3(s3_url): + if s3_url and ceph: + ceph = urlparse.urlparse(s3_url) + connection = boto.connect_s3( + host=ceph.hostname, + port=ceph.port, + is_secure=ceph.scheme == 'https', + calling_format=OrdinaryCallingFormat(), + **aws_connect_params + ) + flavour = 'ceph' + elif is_fakes3(s3_url): fakes3 = urlparse.urlparse(s3_url) connection = S3Connection( is_secure=fakes3.scheme == 'fakes3s', @@ -376,12 +437,13 @@ def main(): state = module.params.get("state") if state == 'present': - create_bucket(connection, module, location) + create_bucket(connection, module, location, flavour=flavour) elif state == 'absent': - destroy_bucket(connection, module) + destroy_bucket(connection, module, flavour=flavour) from ansible.module_utils.basic import * from ansible.module_utils.ec2 import * +import urlparse if __name__ == '__main__': main() From 78b48296232c01408544b1866f51664d24559c70 Mon Sep 17 00:00:00 2001 From: "Javier M. Mellid" Date: Sat, 18 Jun 2016 00:06:13 +0200 Subject: [PATCH 2/3] Adapt exception syntax to work under python3 in s3_bucket.py Signed-off-by: Javier M. Mellid --- cloud/amazon/s3_bucket.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cloud/amazon/s3_bucket.py b/cloud/amazon/s3_bucket.py index fa2e4b92e64..7d07f3b856d 100644 --- a/cloud/amazon/s3_bucket.py +++ b/cloud/amazon/s3_bucket.py @@ -151,11 +151,11 @@ def _create_bucket(connection, module, location): try: bucket = connection.get_bucket(name) - except S3ResponseError, e: + except S3ResponseError as e: try: bucket = connection.create_bucket(name, location=location) changed = True - except S3CreateError, e: + except S3CreateError as e: module.fail_json(msg=e.message) # Versioning @@ -165,7 +165,7 @@ def _create_bucket(connection, module, location): bucket.configure_versioning(versioning) changed = True versioning_status = bucket.get_versioning_status() - except S3ResponseError, e: + except S3ResponseError as e: module.fail_json(msg=e.message) elif not versioning_status and not versioning: # do nothing @@ -195,7 +195,7 @@ def _create_bucket(connection, module, location): # Policy try: current_policy = bucket.get_policy() - except S3ResponseError, e: + except S3ResponseError as e: if e.error_code == "NoSuchBucketPolicy": current_policy = None else: @@ -210,7 +210,7 @@ def _create_bucket(connection, module, location): bucket.set_policy(policy) changed = True current_policy = bucket.get_policy() - except S3ResponseError, e: + except S3ResponseError as e: module.fail_json(msg=e.message) elif current_policy is None and policy is not None: @@ -220,7 +220,7 @@ def _create_bucket(connection, module, location): bucket.set_policy(policy) changed = True current_policy = bucket.get_policy() - except S3ResponseError, e: + except S3ResponseError as e: module.fail_json(msg=e.message) elif current_policy is not None and policy is None: @@ -228,7 +228,7 @@ def _create_bucket(connection, module, location): bucket.delete_policy() changed = True current_policy = bucket.get_policy() - except S3ResponseError, e: + except S3ResponseError as e: if e.error_code == "NoSuchBucketPolicy": current_policy = None else: @@ -242,7 +242,7 @@ def _create_bucket(connection, module, location): try: current_tags = bucket.get_tags() tag_set = TagSet() - except S3ResponseError, e: + except S3ResponseError as e: if e.error_code == "NoSuchTagSet": current_tags = None else: @@ -263,7 +263,7 @@ def _create_bucket(connection, module, location): bucket.delete_tags() current_tags_dict = tags changed = True - except S3ResponseError, e: + except S3ResponseError as e: module.fail_json(msg=e.message) module.exit_json(changed=changed, name=bucket.name, versioning=versioning_status, requester_pays=requester_pays_status, policy=current_policy, tags=current_tags_dict) @@ -276,7 +276,7 @@ def _destroy_bucket(connection, module): try: bucket = connection.get_bucket(name) - except S3ResponseError, e: + except S3ResponseError as e: if e.error_code != "NoSuchBucket": module.fail_json(msg=e.message) else: @@ -289,13 +289,13 @@ def _destroy_bucket(connection, module): for key in bucket.list(): key.delete() - except BotoServerError, e: + except BotoServerError as e: module.fail_json(msg=e.message) try: bucket = connection.delete_bucket(name) changed = True - except S3ResponseError, e: + except S3ResponseError as e: module.fail_json(msg=e.message) module.exit_json(changed=changed) @@ -308,11 +308,11 @@ def _create_bucket_ceph(connection, module, location): try: bucket = connection.get_bucket(name) - except S3ResponseError, e: + except S3ResponseError as e: try: bucket = connection.create_bucket(name, location=location) changed = True - except S3CreateError, e: + except S3CreateError as e: module.fail_json(msg=e.message) module.exit_json(changed=changed) @@ -426,9 +426,9 @@ def main(): if connection is None: connection = boto.connect_s3(**aws_connect_params) - except boto.exception.NoAuthHandlerFound, e: + except boto.exception.NoAuthHandlerFound as e: module.fail_json(msg='No Authentication Handler found: %s ' % str(e)) - except Exception, e: + except Exception as e: module.fail_json(msg='Failed to connect to S3: %s' % str(e)) if connection is None: # this should never happen From 2c1530b647c95f871aea502a441c7fd66dafd863 Mon Sep 17 00:00:00 2001 From: "Javier M. Mellid" Date: Sat, 18 Jun 2016 00:10:18 +0200 Subject: [PATCH 3/3] Imports should be near the top of the file Prior to 2.1, imports of module_utils was actually a preprocessor-like substitution. So importing at the bottom helped preserve line numbers when debugging. We'll be moving these to the top of files as time goes on. Signed-off-by: Javier M. Mellid --- cloud/amazon/s3_bucket.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloud/amazon/s3_bucket.py b/cloud/amazon/s3_bucket.py index 7d07f3b856d..30c0e154242 100644 --- a/cloud/amazon/s3_bucket.py +++ b/cloud/amazon/s3_bucket.py @@ -109,6 +109,9 @@ EXAMPLES = ''' import xml.etree.ElementTree as ET import urlparse +from ansible.module_utils.basic import * +from ansible.module_utils.ec2 import * + try: import boto.ec2 from boto.s3.connection import OrdinaryCallingFormat, Location @@ -441,9 +444,5 @@ def main(): elif state == 'absent': destroy_bucket(connection, module, flavour=flavour) -from ansible.module_utils.basic import * -from ansible.module_utils.ec2 import * -import urlparse - if __name__ == '__main__': main()