cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #1599 from jmunhoz/s3-bucket-ceph

Browse source 
Add Ceph RGW S3 compatibility
This commit is contained in:
Ryan Brown 2016-06-27 13:14:07 -04:00 committed by GitHub
commit ac8e3f18a3
1 changed files with 88 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
cloud/amazon/s3_bucket.py
View file

@ -16,9 +16,9 @@
DOCUMENTATION = ''' DOCUMENTATION = '''
--- ---
module: s3_bucket module: s3_bucket
short_description: Manage s3 buckets in AWS short_description: Manage S3 buckets in AWS, Ceph, Walrus and FakeS3
description: description:
- Manage s3 buckets in AWS - Manage S3 buckets in AWS, Ceph, Walrus and FakeS3
version_added: "2.0" version_added: "2.0"
author: "Rob White (@wimnat)" author: "Rob White (@wimnat)"
options: options:
@ -40,9 +40,13 @@ options:
default: null default: null
s3_url: s3_url:
description: description:
- S3 URL endpoint for usage with Eucalypus, fakes3, etc. Otherwise assumes AWS - S3 URL endpoint for usage with Ceph, Eucalypus, fakes3, etc. Otherwise assumes AWS
default: null default: null
aliases: [ S3_URL ] aliases: [ S3_URL ]
ceph:
description:
- Enable API compatibility with Ceph. It takes into account the S3 API subset working with Ceph in order to provide the same module behaviour where possible.
version_added: "2.2"
requester_pays: requester_pays:
description: description:
- With Requester Pays buckets, the requester instead of the bucket owner pays the cost of the request and the data download from the bucket. - With Requester Pays buckets, the requester instead of the bucket owner pays the cost of the request and the data download from the bucket.
@ -78,6 +82,12 @@ EXAMPLES = '''
- s3_bucket: - s3_bucket:
name: mys3bucket name: mys3bucket
# Create a simple s3 bucket on Ceph Rados Gateway
- s3_bucket:
name: mys3bucket
s3_url: http://your-ceph-rados-gateway-server.xxx
ceph: true
# Remove an s3 bucket and any keys it contains # Remove an s3 bucket and any keys it contains
- s3_bucket: - s3_bucket:
name: mys3bucket name: mys3bucket
@ -99,6 +109,9 @@ EXAMPLES = '''
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import urlparse import urlparse
from ansible.module_utils.basic import *
from ansible.module_utils.ec2 import *
try: try:
import boto.ec2 import boto.ec2
from boto.s3.connection import OrdinaryCallingFormat, Location from boto.s3.connection import OrdinaryCallingFormat, Location
@ -130,8 +143,8 @@ def create_tags_container(tags):
tags_obj.add_tag_set(tag_set) tags_obj.add_tag_set(tag_set)
return tags_obj return tags_obj
def create_bucket(connection, module, location): def _create_bucket(connection, module, location):
policy = module.params.get("policy") policy = module.params.get("policy")
name = module.params.get("name") name = module.params.get("name")
requester_pays = module.params.get("requester_pays") requester_pays = module.params.get("requester_pays")
@ -141,11 +154,11 @@ def create_bucket(connection, module, location):
try: try:
bucket = connection.get_bucket(name) bucket = connection.get_bucket(name)
except S3ResponseError, e: except S3ResponseError as e:
try: try:
bucket = connection.create_bucket(name, location=location) bucket = connection.create_bucket(name, location=location)
changed = True changed = True
except S3CreateError, e: except S3CreateError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
# Versioning # Versioning
@ -155,7 +168,7 @@ def create_bucket(connection, module, location):
bucket.configure_versioning(versioning) bucket.configure_versioning(versioning)
changed = True changed = True
versioning_status = bucket.get_versioning_status() versioning_status = bucket.get_versioning_status()
except S3ResponseError, e: except S3ResponseError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
elif not versioning_status and not versioning: elif not versioning_status and not versioning:
# do nothing # do nothing
@ -185,7 +198,7 @@ def create_bucket(connection, module, location):
# Policy # Policy
try: try:
current_policy = bucket.get_policy() current_policy = bucket.get_policy()
except S3ResponseError, e: except S3ResponseError as e:
if e.error_code == "NoSuchBucketPolicy": if e.error_code == "NoSuchBucketPolicy":
current_policy = None current_policy = None
else: else:
@ -200,7 +213,7 @@ def create_bucket(connection, module, location):
bucket.set_policy(policy) bucket.set_policy(policy)
changed = True changed = True
current_policy = bucket.get_policy() current_policy = bucket.get_policy()
except S3ResponseError, e: except S3ResponseError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
elif current_policy is None and policy is not None: elif current_policy is None and policy is not None:
@ -210,7 +223,7 @@ def create_bucket(connection, module, location):
bucket.set_policy(policy) bucket.set_policy(policy)
changed = True changed = True
current_policy = bucket.get_policy() current_policy = bucket.get_policy()
except S3ResponseError, e: except S3ResponseError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
elif current_policy is not None and policy is None: elif current_policy is not None and policy is None:
@ -218,7 +231,7 @@ def create_bucket(connection, module, location):
bucket.delete_policy() bucket.delete_policy()
changed = True changed = True
current_policy = bucket.get_policy() current_policy = bucket.get_policy()
except S3ResponseError, e: except S3ResponseError as e:
if e.error_code == "NoSuchBucketPolicy": if e.error_code == "NoSuchBucketPolicy":
current_policy = None current_policy = None
else: else:
@ -232,7 +245,7 @@ def create_bucket(connection, module, location):
try: try:
current_tags = bucket.get_tags() current_tags = bucket.get_tags()
tag_set = TagSet() tag_set = TagSet()
except S3ResponseError, e: except S3ResponseError as e:
if e.error_code == "NoSuchTagSet": if e.error_code == "NoSuchTagSet":
current_tags = None current_tags = None
else: else:
@ -253,12 +266,12 @@ def create_bucket(connection, module, location):
bucket.delete_tags() bucket.delete_tags()
current_tags_dict = tags current_tags_dict = tags
changed = True changed = True
except S3ResponseError, e: except S3ResponseError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
module.exit_json(changed=changed, name=bucket.name, versioning=versioning_status, requester_pays=requester_pays_status, policy=current_policy, tags=current_tags_dict) module.exit_json(changed=changed, name=bucket.name, versioning=versioning_status, requester_pays=requester_pays_status, policy=current_policy, tags=current_tags_dict)
def destroy_bucket(connection, module): def _destroy_bucket(connection, module):
force = module.params.get("force") force = module.params.get("force")
name = module.params.get("name") name = module.params.get("name")
@ -266,7 +279,7 @@ def destroy_bucket(connection, module):
try: try:
bucket = connection.get_bucket(name) bucket = connection.get_bucket(name)
except S3ResponseError, e: except S3ResponseError as e:
if e.error_code != "NoSuchBucket": if e.error_code != "NoSuchBucket":
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
else: else:
@ -279,17 +292,50 @@ def destroy_bucket(connection, module):
for key in bucket.list(): for key in bucket.list():
key.delete() key.delete()
except BotoServerError, e: except BotoServerError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
try: try:
bucket = connection.delete_bucket(name) bucket = connection.delete_bucket(name)
changed = True changed = True
except S3ResponseError, e: except S3ResponseError as e:
module.fail_json(msg=e.message) module.fail_json(msg=e.message)
module.exit_json(changed=changed) module.exit_json(changed=changed)
def _create_bucket_ceph(connection, module, location):
name = module.params.get("name")
changed = False
try:
bucket = connection.get_bucket(name)
except S3ResponseError as e:
try:
bucket = connection.create_bucket(name, location=location)
changed = True
except S3CreateError as e:
module.fail_json(msg=e.message)
module.exit_json(changed=changed)
def _destroy_bucket_ceph(connection, module):
_destroy_bucket(connection, module)
def create_bucket(connection, module, location, flavour='aws'):
if flavour == 'ceph':
_create_bucket_ceph(connection, module, location)
else:
_create_bucket(connection, module, location)
def destroy_bucket(connection, module, flavour='aws'):
if flavour == 'ceph':
_destroy_bucket_ceph(connection, module)
else:
_destroy_bucket(connection, module)
def is_fakes3(s3_url): def is_fakes3(s3_url):
""" Return True if s3_url has scheme fakes3:// """ """ Return True if s3_url has scheme fakes3:// """
if s3_url is not None: if s3_url is not None:
@ -319,7 +365,8 @@ def main():
s3_url = dict(aliases=['S3_URL']), s3_url = dict(aliases=['S3_URL']),
state = dict(default='present', choices=['present', 'absent']), state = dict(default='present', choices=['present', 'absent']),
tags = dict(required=None, default={}, type='dict'), tags = dict(required=None, default={}, type='dict'),
versioning = dict(default='no', type='bool') versioning = dict(default='no', type='bool'),
ceph = dict(default='no', type='bool')
) )
) )
@ -344,10 +391,27 @@ def main():
if not s3_url and 'S3_URL' in os.environ: if not s3_url and 'S3_URL' in os.environ:
s3_url = os.environ['S3_URL'] s3_url = os.environ['S3_URL']
ceph = module.params.get('ceph')
if ceph and not s3_url:
module.fail_json(msg='ceph flavour requires s3_url')
flavour = 'aws'
# Look at s3_url and tweak connection settings # Look at s3_url and tweak connection settings
# if connecting to Walrus or fakes3 # if connecting to Walrus or fakes3
try: try:
if is_fakes3(s3_url): if s3_url and ceph:
ceph = urlparse.urlparse(s3_url)
connection = boto.connect_s3(
host=ceph.hostname,
port=ceph.port,
is_secure=ceph.scheme == 'https',
calling_format=OrdinaryCallingFormat(),
**aws_connect_params
)
flavour = 'ceph'
elif is_fakes3(s3_url):
fakes3 = urlparse.urlparse(s3_url) fakes3 = urlparse.urlparse(s3_url)
connection = S3Connection( connection = S3Connection(
is_secure=fakes3.scheme == 'fakes3s', is_secure=fakes3.scheme == 'fakes3s',
@ -365,9 +429,9 @@ def main():
if connection is None: if connection is None:
connection = boto.connect_s3(**aws_connect_params) connection = boto.connect_s3(**aws_connect_params)
except boto.exception.NoAuthHandlerFound, e: except boto.exception.NoAuthHandlerFound as e:
module.fail_json(msg='No Authentication Handler found: %s ' % str(e)) module.fail_json(msg='No Authentication Handler found: %s ' % str(e))
except Exception, e: except Exception as e:
module.fail_json(msg='Failed to connect to S3: %s' % str(e)) module.fail_json(msg='Failed to connect to S3: %s' % str(e))
if connection is None: # this should never happen if connection is None: # this should never happen
@ -376,12 +440,9 @@ def main():
state = module.params.get("state") state = module.params.get("state")
if state == 'present': if state == 'present':
create_bucket(connection, module, location) create_bucket(connection, module, location, flavour=flavour)
elif state == 'absent': elif state == 'absent':
destroy_bucket(connection, module) destroy_bucket(connection, module, flavour=flavour)
from ansible.module_utils.basic import *
from ansible.module_utils.ec2 import *
if __name__ == '__main__': if __name__ == '__main__':
main() main()