Cloudscale volume (#49887)

* Cloudscale integration test setup

CloudProvider and CloudEnvironment classes for Cloudscale integration
tests. This also contains a cloudscale_common role with common
variables for all tests.

* cloudscale_volume module

New cloud module to manage volumes on the cloudscale.ch IaaS service.

lib/ansible/module_utils/cloudscale.py

@@ -42,7 +42,7 @@ class AnsibleCloudscaleBase(object):
             self._module.fail_json(msg='Failure while calling the cloudscale.ch API with GET for '
                                        '"%s".' % api_call, fetch_url_info=info)
 
-    def _post(self, api_call, data=None):
+    def _post_or_patch(self, api_call, method, data):
         headers = self._auth_header.copy()
         if data is not None:
             data = self._module.jsonify(data)
@@ -51,7 +51,7 @@ class AnsibleCloudscaleBase(object):
         resp, info = fetch_url(self._module,
                                API_URL + api_call,
                                headers=headers,
-                               method='POST',
+                               method=method,
                                data=data,
                                timeout=self._module.params['api_timeout'])
 
@@ -60,8 +60,14 @@ class AnsibleCloudscaleBase(object):
         elif info['status'] == 204:
             return None
         else:
-            self._module.fail_json(msg='Failure while calling the cloudscale.ch API with POST for '
+            self._module.fail_json(msg='Failure while calling the cloudscale.ch API with %s for '
-                                       '"%s".' % api_call, fetch_url_info=info)
+                                       '"%s".' % (method, api_call), fetch_url_info=info)
+
+    def _post(self, api_call, data=None):
+        return self._post_or_patch(api_call, 'POST', data)
+
+    def _patch(self, api_call, data=None):
+        return self._post_or_patch(api_call, 'PATCH', data)
 
     def _delete(self, api_call):
         resp, info = fetch_url(self._module,

lib/ansible/modules/cloud/cloudscale/cloudscale_volume.py

@@ -0,0 +1,296 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# (c) 2018, Gaudenz Steinlin <gaudenz.steinlin@cloudscale.ch>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+
+DOCUMENTATION = '''
+---
+module: cloudscale_volume
+short_description: Manages volumes on the cloudscale.ch IaaS service
+description:
+  - Create, attach/detach and delete volumes on the cloudscale.ch IaaS service.
+  - All operations are performed using the cloudscale.ch public API v1.
+  - "For details consult the full API documentation:
+     U(https://www.cloudscale.ch/en/api/v1)."
+  - A valid API token is required for all operations. You can create as many
+    tokens as you like using the cloudscale.ch control panel at
+    U(https://control.cloudscale.ch).
+notes:
+  - Instead of the I(api_token) parameter the C(CLOUDSCALE_API_TOKEN) environment
+    variable can be used.
+  - To create a new volume at least the I(name) and I(size_gb) options
+    are required.
+  - A volume can be created and attached to a server in the same task.
+version_added: 2.8
+author: "Gaudenz Steinlin (@gaudenz)"
+options:
+  state:
+    description:
+      - State of the volume.
+    default: present
+    choices: [ present, absent ]
+  name:
+    description:
+      - Name of the volume. Either name or UUID must be present to change an
+        existing volume.
+  uuid:
+    description:
+      - UUID of the volume. Either name or UUID must be present to change an
+        existing volume.
+  size_gb:
+    description:
+      - Size of the volume in GB.
+  type:
+    description:
+      - Type of the volume. Cannot be changed after creating the volume.
+        Defaults to ssd on volume creation.
+    choices: [ ssd, bulk ]
+  server_uuids:
+    description:
+      - UUIDs of the servers this volume is attached to. Set this to [] to
+        detach the volume. Currently a volume can only be attached to a
+        single server.
+    aliases: [ server_uuid ]
+  api_token:
+    description:
+      - cloudscale.ch API token.
+      - This can also be passed in the C(CLOUDSCALE_API_TOKEN) environment
+        variable.
+  api_timeout:
+    description:
+      - Timeout in seconds for calls to the cloudscale.ch API.
+    default: 30
+'''
+
+EXAMPLES = '''
+# Create a new SSD volume
+- name: Create an SSD volume
+  cloudscale_volume:
+    name: my_ssd_volume
+    size_gb: 50
+    api_token: xxxxxx
+  register: my_ssd_volume
+
+# Attach an existing volume to a server
+- name: Attach volume to server
+  cloudscale_volume:
+    uuid: my_ssd_volume.uuid
+    server_uuids:
+      - ea3b39a3-77a8-4d0b-881d-0bb00a1e7f48
+    api_token: xxxxxx
+
+# Create and attach a volume to a server
+- name: Create and attach volume to server
+  cloudscale_volume:
+    name: my_ssd_volume
+    size_gb: 50
+    server_uuids:
+      - ea3b39a3-77a8-4d0b-881d-0bb00a1e7f48
+    api_token: xxxxxx
+
+# Detach volume from server
+- name: Detach volume from server
+  cloudscale_volume:
+    uuid: my_ssd_volume.uuid
+    server_uuids: []
+    api_token: xxxxxx
+
+# Delete a volume
+- name: Delete volume
+  cloudscale_volume:
+    name: my_ssd_volume
+    state: absent
+    api_token: xxxxxx
+'''
+
+RETURN = '''
+href:
+  description: The API URL to get details about this volume.
+  returned: success when state == present
+  type: str
+  sample: https://api.cloudscale.ch/v1/volumes/2db69ba3-1864-4608-853a-0771b6885a3a
+uuid:
+  description: The unique identifier for this volume.
+  returned: success when state == present
+  type: str
+  sample: 2db69ba3-1864-4608-853a-0771b6885a3a
+name:
+  description: The display name of the volume.
+  returned: success when state == present
+  type: str
+  sample: my_ssd_volume
+size_gb:
+  description: The size of the volume in GB.
+  returned: success when state == present
+  type: str
+  sample: 50
+type:
+  description: "The type of the volume. There are currently two options:
+                ssd (default) or bulk."
+  returned: success when state == present
+  type: str
+  sample: bulk
+server_uuids:
+  description: The UUIDs of the servers this volume is attached to.
+  returned: success when state == present
+  type: list
+  sample: ['47cec963-fcd2-482f-bdb6-24461b2d47b1']
+state:
+  description: The current status of the volume.
+  returned: success
+  type: str
+  sample: present
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.cloudscale import (AnsibleCloudscaleBase,
+                                             cloudscale_argument_spec,
+                                             )
+
+
+class AnsibleCloudscaleVolume(AnsibleCloudscaleBase):
+
+    def __init__(self, module):
+        super(AnsibleCloudscaleVolume, self).__init__(module)
+        params = self._module.params
+        self.info = {
+            'name': params['name'],
+            'uuid': params['uuid'],
+            'state': 'absent',
+        }
+        self.changed = False
+        if params['uuid'] is not None:
+            vol = self._get('volumes/%s' % params['uuid'])
+            if vol is None and params['state'] == 'present':
+                self._module.fail_json(
+                    msg='Volume with UUID %s does not exist. Can\'t create a '
+                        'volume with a predefined UUID.' % params['uuid'],
+                )
+            elif vol is not None:
+                self.info = vol
+                self.info['state'] = 'present'
+        else:
+            resp = self._get('volumes')
+            volumes = [vol for vol in resp if vol['name'] == params['name']]
+            if len(volumes) == 1:
+                self.info = volumes[0]
+                self.info['state'] = 'present'
+            elif len(volumes) > 1:
+                self._module.fail_json(
+                    msg='More than 1 volume with name "%s" exists.'
+                        % params['name'],
+                )
+
+    def create(self):
+        params = self._module.params
+
+        # check for required parameters to create a volume
+        missing_parameters = []
+        for p in ('name', 'size_gb'):
+            if p not in params or not params[p]:
+                missing_parameters.append(p)
+
+        if len(missing_parameters) > 0:
+            self._module.fail_json(
+                msg='Missing required parameter(s) to create a volume: %s.'
+                    % ' '.join(missing_parameters),
+            )
+
+        data = {
+            'name': params['name'],
+            'size_gb': params['size_gb'],
+            'type': params['type'] or 'ssd',
+            'server_uuids': params['server_uuids'] or [],
+        }
+
+        self.info = self._post('volumes', data)
+        self.info['state'] = 'present'
+        self.changed = True
+
+    def delete(self):
+        self._delete('volumes/%s' % self.info['uuid'])
+        self.info = {
+            'name': self.info['name'],
+            'uuid': self.info['uuid'],
+            'state': 'absent',
+        }
+        self.changed = True
+
+    def update(self, param):
+        self._patch(
+            'volumes/%s' % self.info['uuid'],
+            {param: self._module.params[param]},
+        )
+        self.info[param] = self._module.params[param]
+        self.changed = True
+
+
+def main():
+    argument_spec = cloudscale_argument_spec()
+    argument_spec.update(dict(
+        state=dict(default='present', choices=('present', 'absent')),
+        name=dict(),
+        uuid=dict(),
+        size_gb=dict(type='int'),
+        type=dict(choices=('ssd', 'bulk')),
+        server_uuids=dict(type='list', aliases=['server_uuid']),
+    ))
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        required_one_of=(('name', 'uuid'),),
+        mutually_exclusive=(('name', 'uuid'),),
+        supports_check_mode=True,
+    )
+
+    volume = AnsibleCloudscaleVolume(module)
+    if module.check_mode:
+        changed = False
+        for param, conv in (('state', str),
+                            ('server_uuids', set),
+                            ('size_gb', int)):
+            if module.params[param] is None:
+                continue
+
+            if conv(volume.info[param]) != conv(module.params[param]):
+                changed = True
+                break
+
+        module.exit_json(changed=changed,
+                         **volume.info)
+
+    if (volume.info['state'] == 'absent'
+       and module.params['state'] == 'present'):
+        volume.create()
+    elif (volume.info['state'] == 'present'
+          and module.params['state'] == 'absent'):
+        volume.delete()
+
+    if module.params['state'] == 'present':
+        for param, conv in (('server_uuids', set), ('size_gb', int)):
+            if module.params[param] is None:
+                continue
+            if conv(volume.info[param]) != conv(module.params[param]):
+                volume.update(param)
+
+        if (module.params['type'] is not None
+           and volume.info['type'] != module.params['type']):
+            module.fail_json(
+                msg='Cannot change type of an existing volume.',
+            )
+
+    module.exit_json(changed=volume.changed, **volume.info)
+
+
+if __name__ == '__main__':
+    main()

test/integration/cloud-config-cloudscale.ini.template

@@ -0,0 +1,8 @@
+# This is the configuration template for ansible-test cloudscale integration tests.
+#
+# You do not need this template if you are:
+#
+# 1) Running integration tests without using ansible-test.
+#
+[default]
+cloudscale_api_token = @API_TOKEN

test/integration/targets/cloudscale_common/aliases

@@ -0,0 +1 @@
+hidden

test/integration/targets/cloudscale_common/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+# The image to use for test servers
+cloudscale_test_image: 'debian-9'
+
+# The flavor to use for test servers
+cloudscale_test_flavor: 'flex-2'
+
+# SSH key to use for test servers
+cloudscale_test_ssh_key: |
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible

test/integration/targets/cloudscale_volume/aliases

@@ -0,0 +1,2 @@
+cloud/cloudscale
+unsupported

test/integration/targets/cloudscale_volume/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - cloudscale_common

test/integration/targets/cloudscale_volume/tasks/check-mode.yml

@@ -0,0 +1,62 @@
+---
+- name: Create volume in check mode
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-check-mode'
+    size_gb: 50
+  register: check_mode_vol
+  check_mode: True
+- name: Delete volume created in check mode
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-check-mode'
+    state: 'absent'
+  register: check_mode_delete
+- name: 'VERIFY: Create volume in check mode'
+  assert:
+    that:
+      - check_mode_vol is successful
+      - check_mode_vol is changed
+      - check_mode_delete is successful
+      - check_mode_delete is not changed
+
+- name: Create volume
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    size_gb: 50
+- name: Attach volume in check mode
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    server_uuids:
+      - '{{ server.uuid }}'
+  check_mode: True
+  register: check_mode_attach
+- name: Detach volume
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    server_uuids: []
+  register: check_mode_detach
+- name: 'VERIFY: Attach volume in check mode'
+  assert:
+    that:
+      - check_mode_attach is successful
+      - check_mode_attach is changed
+      - check_mode_detach is successful
+      - check_mode_detach is not changed
+
+- name: Resize volume in check mode
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    size_gb: 100
+  register: check_mode_resize
+  check_mode: True
+- name: Get volume info
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+  register: check_mode_info
+- name: 'VERIFY: Resize volume in check mode'
+  assert:
+    that:
+      - check_mode_resize is successful
+      - check_mode_resize is changed
+      - check_mode_info is successful
+      - check_mode_info is not changed
+      - check_mode_info.size_gb == 50

test/integration/targets/cloudscale_volume/tasks/cleanup.yml

@@ -0,0 +1,5 @@
+---
+- name: Remove test server
+  cloudscale_server:
+    uuid: '{{ server.uuid }}'
+    state: 'absent'

test/integration/targets/cloudscale_volume/tasks/failures.yml

@@ -0,0 +1,63 @@
+---
+- name: Create two volumes with the same name
+  uri:
+    url: 'https://api.cloudscale.ch/v1/volumes'
+    method: POST
+    headers:
+      Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}'
+    body:
+      name: '{{ cloudscale_resource_prefix }}-duplicate'
+      size_gb: 50
+    body_format: json
+    status_code: 201
+  register: duplicate
+  with_sequence: count=2
+
+- name: Try access to duplicate name
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-duplicate'
+    size_gb: 10
+  register: vol
+  ignore_errors: True
+- name: 'VERIFY: Try access to duplicate name'
+  assert:
+    that:
+      - vol is failed
+
+- name: Delete duplicate volumes by UUID
+  cloudscale_volume:
+    uuid: '{{ item.json.uuid }}'
+    state: 'absent'
+  with_items: '{{ duplicate.results }}'
+
+- name: Fail volume creation with UUID
+  cloudscale_volume:
+    uuid: ea3b39a3-77a8-4d0b-881d-0bb00a1e7f48
+    size_gb: 10
+  register: vol
+  ignore_errors: True
+- name: 'VERIFY: Fail volume creation with UUID'
+  assert:
+    that:
+      - vol is failed
+
+- name: Create volume
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-name-UUID'
+    size_gb: 50
+  register: vol
+- name: Fail name and UUID
+  cloudscale_volume:
+    name: '{{ vol.name }}'
+    uuid: '{{ vol.uuid }}'
+    size_gb: 100
+  register: vol
+  ignore_errors: True
+- name: 'VERIFY: Fail name and UUID'
+  assert:
+    that:
+      - vol is failed
+- name: 'Delete volume'
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-name-UUID'
+    state: 'absent'

test/integration/targets/cloudscale_volume/tasks/main.yml

@@ -0,0 +1,158 @@
+---
+- import_tasks: setup.yml
+
+- name: Create volume
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    size_gb: 50
+  register: vol
+- name: 'VERIFY: Create volume'
+  assert:
+    that:
+      - vol is successful
+      - vol is changed
+      - vol.size_gb == 50
+      - vol.name == '{{ cloudscale_resource_prefix }}-vol'
+
+- name: Create volume indempotence
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    size_gb: 50
+  register: vol
+- name: 'VERIFY: Create volume indempotence'
+  assert:
+    that:
+      - vol is successful
+      - vol is not changed
+
+- name: Attach existing volume by name to server
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    server_uuids:
+      - '{{ server.uuid }}'
+  register: vol
+- name: 'VERIFY: Attach existing volume by name to server'
+  assert:
+    that:
+      - vol is successful
+      - vol is changed
+      - server.uuid in vol.server_uuids
+
+- name: Attach existing volume by name to server indempotence
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-vol'
+    server_uuids:
+      - '{{ server.uuid }}'
+  register: vol
+- name: 'VERIFY: Attach existing volume by name to server indempotence'
+  assert:
+    that:
+      - vol is successful
+      - vol is not changed
+      - server.uuid in vol.server_uuids
+
+- name: Resize attached volume by UUID
+  cloudscale_volume:
+    uuid: '{{ vol.uuid }}'
+    size_gb: 100
+  register: vol
+- name: 'VERIFY: Resize attached volume by UUID'
+  assert:
+    that:
+      - vol is successful
+      - vol is changed
+      - vol.size_gb == 100
+
+- name: Resize attached volume by UUID indempotence
+  cloudscale_volume:
+    uuid: '{{ vol.uuid }}'
+    size_gb: 100
+  register: vol
+- name: 'VERIFY: Resize attached volume by UUID indempotence'
+  assert:
+    that:
+      - vol is successful
+      - vol is not changed
+      - vol.size_gb == 100
+
+- name: Delete attached volume by UUID
+  cloudscale_volume:
+    uuid: '{{ vol.uuid }}'
+    state: 'absent'
+  register: deleted
+- name: 'VERIFY: Delete attached volume by UUID'
+  assert:
+    that:
+      - deleted is successful
+      - deleted is changed
+      - deleted.state == 'absent'
+
+- name: Delete attached volume by UUID indempotence
+  cloudscale_volume:
+    uuid: '{{ vol.uuid }}'
+    state: 'absent'
+  register: deleted
+- name: 'VERIFY: Delete attached volume by UUID indempotence'
+  assert:
+    that:
+      - deleted is successful
+      - deleted is not changed
+      - deleted.state == 'absent'
+
+- name: Create bulk volume and attach
+  cloudscale_volume:
+    name: '{{ cloudscale_resource_prefix }}-bulk'
+    type: bulk
+    size_gb: 100
+    server_uuids:
+      - '{{ server.uuid }}'
+  register: bulk
+- name: 'VERIFY: Create bulk volume and attach'
+  assert:
+    that:
+      - bulk is successful
+      - bulk is changed
+      - bulk.size_gb == 100
+      - server.uuid in bulk.server_uuids
+
+- name: Detach volume by UUID
+  cloudscale_volume:
+    uuid: '{{ bulk.uuid }}'
+    server_uuids: []
+  register: bulk
+- name: 'VERIFY: Detach volume by UUID'
+  assert:
+    that:
+      - bulk is successful
+      - bulk is changed
+      - bulk.server_uuids == []
+
+- name: Resize detached volume by name
+  cloudscale_volume:
+    name: '{{ bulk.name }}'
+    size_gb: 200
+  register: bulk
+- name: 'VERIFY: Resize detached volume by name'
+  assert:
+    that:
+      - bulk is successful
+      - bulk is changed
+      - bulk.size_gb == 200
+
+- name: Delete volume by name
+  cloudscale_volume:
+    name: '{{ bulk.name }}'
+    state: 'absent'
+  register: bulk
+- name: 'VERIFY: Delete volume by name'
+  assert:
+    that:
+      - bulk is successful
+      - bulk is changed
+      - bulk.state == 'absent'
+
+- import_tasks: failures.yml
+
+- import_tasks: check-mode.yml
+
+- import_tasks: cleanup.yml

test/integration/targets/cloudscale_volume/tasks/setup.yml

@@ -0,0 +1,8 @@
+---
+- name: Create test instance
+  cloudscale_server:
+    name: '{{ cloudscale_resource_prefix }}-server'
+    flavor: '{{ cloudscale_test_flavor }}'
+    image: '{{ cloudscale_test_image }}'
+    ssh_keys: '{{ cloudscale_test_ssh_key }}'
+  register: server

test/runner/lib/cloud/cloudscale.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# (c) 2018, Gaudenz Steinlin <gaudenz.steinlin@cloudscale.ch>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+"""Cloudscale plugin for integration tests."""
+from __future__ import absolute_import, print_function
+
+from os.path import isfile
+
+from lib.cloud import (
+    CloudProvider,
+    CloudEnvironment,
+)
+
+from lib.util import ConfigParser, display
+
+
+class CloudscaleCloudProvider(CloudProvider):
+    """Cloudscale cloud provider plugin. Sets up cloud resources before
+       delegation.
+    """
+
+    def __init__(self, args):
+        """
+        :type args: TestConfig
+        """
+        super(CloudscaleCloudProvider, self).__init__(args, config_extension='.ini')
+
+    def filter(self, targets, exclude):
+        """Filter out the cloud tests when the necessary config and resources are not available.
+        :type targets: tuple[TestTarget]
+        :type exclude: list[str]
+        """
+        if isfile(self.config_static_path):
+            return
+
+        super(CloudscaleCloudProvider, self).filter(targets, exclude)
+
+    def setup(self):
+        super(CloudscaleCloudProvider, self).setup()
+
+        if isfile(self.config_static_path):
+            display.info('Using existing %s cloud config: %s'
+                         % (self.platform, self.config_static_path),
+                         verbosity=1)
+            self.config_path = self.config_static_path
+            self.managed = False
+            return True
+
+        return False
+
+
+class CloudscaleCloudEnvironment(CloudEnvironment):
+    """Cloudscale cloud environment plugin. Updates integration test environment
+       after delegation.
+    """
+    def configure_environment(self, env, cmd):
+        """
+        :type env: dict[str, str]
+        :type cmd: list[str]
+        """
+        parser = ConfigParser()
+        parser.read(self.config_path)
+
+        changes = dict(
+            CLOUDSCALE_API_TOKEN=parser.get('default', 'cloudscale_api_token'),
+        )
+
+        env.update(changes)
+
+        cmd.append('-e')
+        cmd.append('cloudscale_resource_prefix=%s' % self.resource_prefix)