Merge pull request #5496 from angstwad/cf-module

Module support for Rackspace Cloud Files

library/cloud/rax_files_objects

@@ -0,0 +1,600 @@
+#!/usr/bin/python -tt
+
+# (c) 2013, Paul Durivage <paul.durivage@rackspace.com>
+#
+# This file is part of Ansible.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = '''
+---
+module:
+short_description: create, fetch, and delete objects in Rackspace Cloud Files
+description:
+  - Upload, download, and delete objects in Rackspace Cloud Files
+version_added: "1.5"
+options:
+  api_key:
+    description:
+      - Rackspace API key (overrides I(credentials))requirements: [ "pyrax" ]
+  clear_meta:
+    description:
+      - Optionally clear existing metadata when applying metadata to existing objects.
+      - Selecting this option is only appropriate when setting typ=meta
+    choices: ["yes", "no"]
+    default: "no"
+  container:
+    description:
+      - The container to use for file object operations.
+    required: true
+  credentials:
+    description:
+      - File to find the Rackspace credentials in (ignored if I(api_key) and
+        I(username) are provided)
+    default: null
+    aliases: ['creds_file']
+  dest:
+    description:
+      - The destination of a "get" operation; i.e. a local directory, "/home/user/myfolder"
+      - Used to specify the destination of an operation on a remote object; i.e. a file name,
+      "file1", or a comma-separated list of remote objects, "file1,file2,file17"
+  expires:
+    description:
+      - Used to set an expiration on a file or folder uploaded to Cloud Files
+      - Requires an integer, specifying expiration in seconds
+  meta:
+    description:
+      - A hash of items to set as metadata values on an uploaded file or folder
+  method:
+    description:
+      - The method of operation to be performed: put, get, delete
+      - Put to upload files to Cloud Files
+      - Get to download files from Cloud Files
+      - Delete to delete remote objects in Cloud Files
+    choices: ["get", "put", "delete"]
+    default: "get"
+  region:
+    description:
+      - Region to create an instance in
+    default: DFW
+  src:
+    description:
+      - Source from which to upload files
+      - Used to specify a remote object as a source for an operation, i.e. a file name,
+      "file1", or a comma-separated list of remote objects, "file1,file2,file17"
+      - src and dest are mutually exclusive on remote-only object operations
+  structure:
+    description:
+      - Used to specify whether to maintain nested directory structure when downloading objects from Cloud Files
+      - Setting to false downloads the contents of a container to a single, flat directory
+    choices: ["yes", "no"]
+    default: "yes"
+  typ:
+    description:
+      - Type of object to do work on: metadata object or a file object
+    choices: ["file", "meta"]
+    default: "file"
+  username:
+    description:
+      - Rackspace username (overrides I(credentials))
+requirements: [ "pyrax" ]
+author: Paul Durivage
+notes:
+  - The following environment variables can be used, C(RAX_USERNAME),
+    C(RAX_API_KEY), C(RAX_CREDS_FILE), C(RAX_CREDENTIALS), C(RAX_REGION).
+  - C(RAX_CREDENTIALS) and C(RAX_CREDS_FILE) points to a credentials file
+    appropriate for pyrax. See U(https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating)
+  - C(RAX_USERNAME) and C(RAX_API_KEY) obviate the use of a credentials file
+  - C(RAX_REGION) defines a Rackspace Public Cloud region (DFW, ORD, LON, ...)
+'''
+
+EXAMPLES = '''
+---
+- name: "Test Cloud Files Objects"
+  hosts: local
+  gather_facts: False
+  tasks:
+    - name: "Get objects from test container"
+      rax_files_objects: container=testcont dest=~/Downloads/testcont
+
+    - name: "Get single object from test container"
+      rax_files_objects: container=testcont src=file1 dest=~/Downloads/testcont
+
+    - name: "Get several objects from test container"
+      rax_files_objects: container=testcont src=file1,file2,file3 dest=~/Downloads/testcont
+
+    - name: "Delete one object in test container"
+      rax_files_objects: container=testcont method=delete dest=file1
+
+    - name: "Delete several objects in test container"
+      rax_files_objects: container=testcont method=delete dest=file2,file3,file4
+
+    - name: "Delete all objects in test container"
+      rax_files_objects: container=testcont method=delete
+
+    - name: "Upload all files to test container"
+      rax_files_objects: container=testcont method=put src=~/Downloads/onehundred
+
+    - name: "Upload one file to test container"
+      rax_files_objects: container=testcont method=put src=~/Downloads/testcont/file1
+
+    - name: "Upload one file to test container with metadata"
+      rax_files_objects:
+        container: testcont
+        src: ~/Downloads/testcont/file2
+        method: put
+        meta:
+          testkey: testdata
+          who_uploaded_this: paul.durivage@rackspace.com
+
+    - name: "Upload one file to test container with TTL of 60 seconds"
+      rax_files_objects: container=testcont method=put src=~/Downloads/testcont/file3 expires=60
+
+    - name: "Attempt to get remote object that does not exist"
+      rax_files_objects: container=testcont method=get src=FileThatDoesNotExist.jpg dest=~/Downloads/testcont
+      ignore_errors: yes
+
+    - name: "Attempt to delete remote object that does not exist"
+      rax_files_objects: container=testcont method=delete dest=FileThatDoesNotExist.jpg
+      ignore_errors: yes
+
+- name: "Test Cloud Files Objects Metadata"
+  hosts: local
+  gather_facts: false
+  tasks:
+    - name: "Get metadata on one object"
+      rax_files_objects:  container=testcont type=meta dest=file2
+
+    - name: "Get metadata on several objects"
+      rax_files_objects:  container=testcont type=meta src=file2,file1
+
+    - name: "Set metadata on an object"
+      rax_files_objects:
+        container: testcont
+        type: meta
+        dest: file17
+        meta:
+          key1: value1
+          key2: value2
+        clear_meta: true
+
+    - name: "Verify metadata is set"
+      rax_files_objects:  container=testcont type=meta src=file17
+
+    - name: "Delete metadata"
+      rax_files_objects:
+        container: testcont
+        type: meta
+        dest: file17
+        method: delete
+        meta:
+          key1: ''
+          key2: ''
+
+    - name: "Get metadata on all objects"
+      rax_files_objects:  container=testcont type=meta
+'''
+
+import os
+
+try:
+    import pyrax
+except ImportError, e:
+    print("failed=True msg='pyrax is required for this module'")
+    sys.exit(1)
+
+EXIT_DICT = {'success': False}
+META_PREFIX = 'x-object-meta-'
+
+
+def _get_container(module, cf, container):
+    try:
+        return cf.get_container(container)
+    except pyrax.exc.NoSuchContainer, e:
+        module.fail_json(msg=e.message)
+
+
+def upload(module, cf, container, src, dest, meta, expires):
+    """ Uploads a single object or a folder to Cloud Files Optionally sets an
+    metadata, TTL value (expires), or Content-Disposition and Content-Encoding
+    headers.
+    """
+    c = _get_container(module, cf, container)
+
+    num_objs_before = len(c.get_object_names())
+
+    if not src:
+        module.fail_json(msg='src must be specified when uploading')
+
+    src = os.path.abspath(os.path.expanduser(src))
+    is_dir = os.path.isdir(src)
+
+    if not is_dir and not os.path.isfile(src) or not os.path.exists(src):
+        module.fail_json(msg='src must be a file or a directory')
+    if dest and is_dir:
+        module.fail_json(msg='dest cannot be set when whole '
+                             'directories are uploaded')
+
+    cont_obj = None
+    if dest and not is_dir:
+        try:
+            cont_obj = c.upload_file(src, obj_name=dest, ttl=expires)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+    elif is_dir:
+        try:
+            id, total_bytes = cf.upload_folder(src, container=c.name, ttl=expires)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+
+        while True:
+            bytes = cf.get_uploaded(id)
+            if bytes == total_bytes:
+                break
+            time.sleep(1)
+    else:
+        try:
+            cont_obj = c.upload_file(src, ttl=expires)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+
+    num_objs_after = len(c.get_object_names())
+
+    if not meta:
+        meta = {}
+
+    meta_result = {}
+    if meta:
+        if cont_obj:
+            meta_result = cont_obj.set_metadata(meta)
+        else:
+            def _set_meta(objs, meta):
+                """ Sets metadata on a list of objects specified by name """
+                for obj in objs:
+                    try:
+                        result = c.get_object(obj).set_metadata(meta)
+                    except Exception, e:
+                        module.fail_json(msg=e.message)
+                    else:
+                        meta_result[obj] = result
+                return meta_result
+
+            def _walker(objs, path, filenames):
+                """ Callback func for os.path.walk  """
+                prefix = ''
+                if path != src:
+                    prefix = path.split(src)[-1].lstrip('/')
+                filenames = [os.path.join(prefix, name) for name in filenames
+                             if not os.path.isdir(name)]
+                objs += filenames
+
+            _objs = []
+            os.path.walk(src, _walker, _objs)
+            meta_result = _set_meta(_objs, meta)
+
+    EXIT_DICT['success'] = True
+    EXIT_DICT['container'] = c.name
+    EXIT_DICT['msg'] = "Uploaded %s to container: %s" % (src, c.name)
+    if cont_obj or locals().get('bytes'):
+        EXIT_DICT['changed'] = True
+    if meta_result:
+        EXIT_DICT['meta'] = {
+            'updated': True
+        }
+
+    if cont_obj:
+        EXIT_DICT['bytes'] = cont_obj.total_bytes
+        EXIT_DICT['etag'] = cont_obj.etag
+    else:
+        EXIT_DICT['bytes'] = total_bytes
+
+    module.exit_json(**EXIT_DICT)
+
+
+def download(module, cf, container, src, dest, structure):
+    """ Download objects from Cloud Files to a local path specified by "dest".
+    Optionally disable maintaining a directory structure by by passing a
+    false value to "structure".
+    """
+    # Looking for an explicit destination
+    if not dest:
+        module.fail_json(msg='dest is a required argument when '
+                             'downloading from Cloud Files')
+
+    # Attempt to fetch the container by name
+    c = _get_container(module, cf, container)
+
+    # Accept a single object name or a comma-separated list of objs
+    # If not specified, get the entire container
+    if src:
+        objs = src.split(',')
+        objs = map(str.strip, objs)
+    else:
+        objs = c.get_object_names()
+
+    dest = os.path.abspath(os.path.expanduser(dest))
+    is_dir = os.path.isdir(dest)
+
+    if not is_dir:
+        module.fail_json(msg='dest must be a directory')
+
+    results = []
+    for obj in objs:
+        try:
+            c.download_object(obj, dest, structure=structure)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+        else:
+            results.append(obj)
+
+    len_results = len(results)
+    len_objs = len(objs)
+
+    EXIT_DICT['container'] = c.name
+    EXIT_DICT['requested_downloaded'] = results
+    if results:
+        EXIT_DICT['changed'] = True
+    if len_results == len_objs:
+        EXIT_DICT['success'] = True
+        EXIT_DICT['msg'] = "%s objects downloaded to %s" % (len_results, dest)
+    else:
+        EXIT_DICT['msg'] = "Error: only %s of %s objects were " \
+                           "downloaded" % (len_results, len_objs)
+    module.exit_json(**EXIT_DICT)
+
+
+def delete(module, cf, container, src, dest):
+    """ Delete specific objects by proving a single file name or a
+    comma-separated list to src OR dest (but not both).  Ommitting file name(s)
+    assumes the entire container is to be deleted.
+    """
+    objs = None
+    if src and dest:
+        module.fail_json(msg="Error: ambiguous instructions; files to be deleted "
+                             "have been specified on both src and dest args")
+    elif dest:
+        objs = dest
+    else:
+        objs = src
+
+    c = _get_container(module, cf, container)
+
+    if objs:
+        objs = objs.split(',')
+        objs = map(str.strip, objs)
+    else:
+        objs = c.get_object_names()
+
+    num_objs = len(objs)
+
+    results = []
+    for obj in objs:
+        try:
+            result = c.delete_object(obj)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+        else:
+            results.append(result)
+
+    num_deleted = results.count(True)
+
+    EXIT_DICT['container'] = c.name
+    EXIT_DICT['deleted'] = num_deleted
+    EXIT_DICT['requested_deleted'] = objs
+
+    if num_deleted:
+        EXIT_DICT['changed'] = True
+
+    if num_objs == num_deleted:
+        EXIT_DICT['success'] = True
+        EXIT_DICT['msg'] = "%s objects deleted" % num_deleted
+    else:
+        EXIT_DICT['msg'] = ("Error: only %s of %s objects "
+                            "deleted" % (num_deleted, num_objs))
+    module.exit_json(**EXIT_DICT)
+
+
+def get_meta(module, cf, container, src, dest):
+    """ Get metadata for a single file, comma-separated list, or entire
+    container
+    """
+    c = _get_container(module, cf, container)
+
+    objs = None
+    if src and dest:
+        module.fail_json(msg="Error: ambiguous instructions; files to be deleted "
+                             "have been specified on both src and dest args")
+    elif dest:
+        objs = dest
+    else:
+        objs = src
+
+    if objs:
+        objs = objs.split(',')
+        objs = map(str.strip, objs)
+    else:
+        objs = c.get_object_names()
+
+    results = {}
+    for obj in objs:
+        try:
+            meta = c.get_object(obj).get_metadata()
+        except Exception, e:
+            module.fail_json(msg=e.message)
+        else:
+            results[obj] = {k.split(META_PREFIX)[-1]: v for k, v in meta.iteritems()}
+
+    EXIT_DICT['container'] = c.name
+    if results:
+        EXIT_DICT['meta_results'] = results
+        EXIT_DICT['success'] = True
+    module.exit_json(**EXIT_DICT)
+
+
+def put_meta(module, cf, container, src, dest, meta, clear_meta):
+    """ Set metadata on a container, single file, or comma-separated list.
+    Passing a true value to clear_meta clears the metadata stored in Cloud
+    Files before setting the new metadata to the value of "meta".
+    """
+
+    objs = None
+    if src and dest:
+        module.fail_json(msg="Error: ambiguous instructions; files to set meta"
+                             " have been specified on both src and dest args")
+    elif dest:
+        objs = dest
+    else:
+        objs = src
+
+    objs = objs.split(',')
+    objs = map(str.strip, objs)
+
+    c = _get_container(module, cf, container)
+
+    results = []
+    for obj in objs:
+        try:
+            result = c.get_object(obj).set_metadata(meta, clear=clear_meta)
+        except Exception, e:
+            module.fail_json(msg=e.message)
+        else:
+            results.append(result)
+
+    EXIT_DICT['container'] = c.name
+    EXIT_DICT['success'] = True
+    if results:
+        EXIT_DICT['changed'] = True
+        EXIT_DICT['num_changed'] = True
+    module.exit_json(**EXIT_DICT)
+
+
+def delete_meta(module, cf, container, src, dest, meta):
+    """ Removes metadata keys and values specified in meta, if any.  Deletes on
+    all objects specified by src or dest (but not both), if any; otherwise it
+    deletes keys on all objects in the container
+    """
+    objs = None
+    if src and dest:
+        module.fail_json(msg="Error: ambiguous instructions; meta keys to be "
+                             "deleted have been specified on both src and dest"
+                             " args")
+    elif dest:
+        objs = dest
+    else:
+        objs = src
+
+    objs = objs.split(',')
+    objs = map(str.strip, objs)
+
+    c = _get_container(module, cf, container)
+
+    results = []  # Num of metadata keys removed, not objects affected
+    for obj in objs:
+        if meta:
+            for k, v in meta.items():
+                try:
+                    result = c.get_object(obj).remove_metadata_key(k)
+                except Exception, e:
+                    module.fail_json(msg=e.message)
+                else:
+                    results.append(result)
+        else:
+            try:
+                o = c.get_object(obj)
+            except pyrax.exc.NoSuchObject, e:
+                module.fail_json(msg=e.message)
+
+            for k, v in o.get_metadata().items():
+                try:
+                    result = o.remove_metadata_key(k)
+                except Exception, e:
+                    module.fail_json(msg=e.message)
+                results.append(result)
+
+    EXIT_DICT['container'] = c.name
+    EXIT_DICT['success'] = True
+    if results:
+        EXIT_DICT['changed'] = True
+        EXIT_DICT['num_deleted'] = len(results)
+    module.exit_json(**EXIT_DICT)
+
+
+def cloudfiles(module, container, src, dest, method, typ, meta, clear_meta,
+               structure, expires):
+        """ Dispatch from here to work with metadata or file objects """
+        cf = pyrax.cloudfiles
+
+        if typ == "file":
+            if method == 'put':
+                upload(module, cf, container, src, dest, meta, expires)
+
+            elif method == 'get':
+                download(module, cf, container, src, dest, structure)
+
+            elif method == 'delete':
+                delete(module, cf, container, src, dest)
+
+        else:
+            if method == 'get':
+                get_meta(module, cf, container, src, dest)
+
+            if method == 'put':
+                put_meta(module, cf, container, src, dest, meta, clear_meta)
+
+            if method == 'delete':
+                delete_meta(module, cf, container, src, dest, meta)
+
+
+def main():
+    argument_spec = rax_argument_spec()
+    argument_spec.update(
+        dict(
+            container=dict(required=True),
+            src=dict(),
+            dest=dict(),
+            method=dict(default='get', choices=['put', 'get', 'delete']),
+            type=dict(default='file', choices=['file', 'meta']),
+            meta=dict(type='dict', default={}),
+            clear_meta=dict(choices=BOOLEANS, default=False, type='bool'),
+            structure=dict(choices=BOOLEANS, default=True, type='bool'),
+            expires=dict(type='int'),
+        )
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        required_together=rax_required_together()
+    )
+
+    container = module.params.get('container')
+    src = module.params.get('src')
+    dest = module.params.get('dest')
+    method = module.params.get('method')
+    typ = module.params.get('type')
+    meta = module.params.get('meta')
+    clear_meta = module.params.get('clear_meta')
+    structure = module.params.get('structure')
+    expires = module.params.get('expires')
+
+    if clear_meta and not typ == 'meta':
+        module.fail_json(msg='clear_meta can only be used when setting metadata')
+
+    setup_rax_module(module, pyrax)
+    cloudfiles(module, container, src, dest, method, typ, meta, clear_meta, structure, expires)
+
+
+from ansible.module_utils.basic import *
+from ansible.module_utils.rax import *
+
+main()