diff --git a/lib/ansible/plugins/inventory/cloudscale.py b/lib/ansible/plugins/inventory/cloudscale.py new file mode 100644 index 00000000000..442799c884f --- /dev/null +++ b/lib/ansible/plugins/inventory/cloudscale.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +# +# Copyright: (c) 2018, Gaudenz Steinlin +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' +--- +name: cloudscale +plugin_type: inventory +author: + - Gaudenz Steinlin (@gaudenz) +short_description: cloudscale.ch inventory source +description: + - Get inventory hosts from cloudscale.ch API +version_added: '2.8' +extends_documentation_fragment: + - constructed +options: + plugin: + description: | + Token that ensures this is a source file for the 'cloudscale' + plugin. + required: True + choices: ['cloudscale'] + inventory_hostname: + description: | + What to register as the inventory hostname. + If set to 'uuid' the uuid of the server will be used and a + group will be created for the server name. + If set to 'name' the name of the server will be used unless + there are more than one server with the same name in which + case the 'uuid' logic will be used. + type: str + choices: + - name + - uuid + default: "name" + ansible_host: + description: | + Which IP address to register as the ansible_host. If the + requested value does not exist or this is set to 'none', no + ansible_host will be set. + type: str + choices: + - public_v4 + - public_v6 + - private + - none + default: public_v4 + api_token: + description: cloudscale.ch API token + env: + - name: CLOUDSCALE_API_TOKEN + type: str + api_timeout: + description: Timeout in seconds for calls to the cloudscale.ch API. + default: 30 + type: int +''' + +EXAMPLES = r''' +# cloudscale_inventory.yml file in YAML format +# Example command line: ansible-inventory --list -i cloudscale_inventory.yml + +plugin: cloudscale +''' + +from collections import defaultdict +from json import loads + +from ansible.errors import AnsibleError +from ansible.module_utils.cloudscale import API_URL +from ansible.module_utils.urls import open_url +from ansible.inventory.group import to_safe_group_name +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable + +iface_type_map = { + 'public_v4': ('public', 4), + 'public_v6': ('public', 6), + 'private': ('private', 4), + 'none': (None, None), +} + + +class InventoryModule(BaseInventoryPlugin, Constructable): + + NAME = 'cloudscale' + + def _get_server_list(self): + # Get list of servers from cloudscale.ch API + response = open_url( + API_URL + '/servers', + headers={'Authorization': 'Bearer %s' % self._token} + ) + return loads(response.read()) + + def verify_file(self, path): + ''' + :param path: the path to the inventory config file + :return the contents of the config file + ''' + if super(InventoryModule, self).verify_file(path): + if path.endswith(('cloudscale.yml', 'cloudscale.yaml')): + return True + self.display.debug( + "cloudscale inventory filename must end with 'cloudscale.yml' or 'cloudscale.yaml'" + ) + return False + + def parse(self, inventory, loader, path, cache=True): + super(InventoryModule, self).parse(inventory, loader, path) + + self._read_config_data(path) + + self._token = self.get_option('api_token') + if not self._token: + raise AnsibleError('Could not find an API token. Set the ' + 'CLOUDSCALE_API_TOKEN environment variable.') + + inventory_hostname = self.get_option('inventory_hostname') + if inventory_hostname not in ('name', 'uuid'): + raise AnsibleError('Invalid value for option inventory_hostname: %s' + % inventory_hostname) + + ansible_host = self.get_option('ansible_host') + if ansible_host not in iface_type_map: + raise AnsibleError('Invalid value for option ansible_host: %s' + % ansible_host) + + # Merge servers with the same name + firstpass = defaultdict(list) + for server in self._get_server_list(): + firstpass[server['name']].append(server) + + # Add servers to inventory + for name, servers in firstpass.items(): + if len(servers) == 1 and inventory_hostname == 'name': + self.inventory.add_host(name) + servers[0]['inventory_hostname'] = name + else: + # Two servers with the same name exist, create a group + # with this name and add the servers by UUID + group_name = to_safe_group_name(name) + if group_name not in self.inventory.groups: + self.inventory.add_group(group_name) + for server in servers: + self.inventory.add_host(server['uuid'], group_name) + server['inventory_hostname'] = server['uuid'] + + # Set variables + iface_type, iface_version = iface_type_map[ansible_host] + for server in servers: + hostname = server.pop('inventory_hostname') + if ansible_host != 'none': + addresses = [address['address'] + for interface in server['interfaces'] + for address in interface['addresses'] + if interface['type'] == iface_type + and address['version'] == iface_version] + + if len(addresses) > 0: + self.inventory.set_variable( + hostname, + 'ansible_host', + addresses[0], + ) + self.inventory.set_variable( + hostname, + 'cloudscale', + server, + ) + + variables = self.inventory.hosts[hostname].get_vars() + # Set composed variables + self._set_composite_vars( + self.get_option('compose'), + variables, + hostname, + self.get_option('strict'), + ) + + # Add host to composed groups + self._add_host_to_composed_groups( + self.get_option('groups'), + variables, + hostname, + self.get_option('strict'), + ) + + # Add host to keyed groups + self._add_host_to_keyed_groups( + self.get_option('keyed_groups'), + variables, + hostname, + self.get_option('strict'), + ) diff --git a/test/integration/targets/cloudscale_common/defaults/main.yml b/test/integration/targets/cloudscale_common/defaults/main.yml index 26ae233c49a..ec567bfffb3 100644 --- a/test/integration/targets/cloudscale_common/defaults/main.yml +++ b/test/integration/targets/cloudscale_common/defaults/main.yml @@ -2,6 +2,9 @@ # The image to use for test servers cloudscale_test_image: 'debian-9' +# Alternate test image to use if a different image is required +cloudscale_alt_test_image: 'ubuntu-18.04' + # The flavor to use for test servers cloudscale_test_flavor: 'flex-2' diff --git a/test/integration/targets/inventory_cloudscale/aliases b/test/integration/targets/inventory_cloudscale/aliases new file mode 100644 index 00000000000..1ce5f091222 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/aliases @@ -0,0 +1,3 @@ +cloud/cloudscale +unsupported +needs/target/cloudscale_common diff --git a/test/integration/targets/inventory_cloudscale/filter_plugins/group_name.py b/test/integration/targets/inventory_cloudscale/filter_plugins/group_name.py new file mode 100644 index 00000000000..42e39b21df8 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/filter_plugins/group_name.py @@ -0,0 +1,14 @@ +from ansible.inventory.group import to_safe_group_name + + +def safe_group_name(name): + return to_safe_group_name(name) + + +class FilterModule(object): + filter_map = { + 'safe_group_name': safe_group_name + } + + def filters(self): + return self.filter_map diff --git a/test/integration/targets/inventory_cloudscale/inventory-private.yml b/test/integration/targets/inventory_cloudscale/inventory-private.yml new file mode 100644 index 00000000000..32000758e02 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/inventory-private.yml @@ -0,0 +1,14 @@ +plugin: cloudscale +ansible_host: private +inventory_hostname: name +groups: + ansible: inventory_hostname.startswith('ansible') + private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0 +keyed_groups: + - prefix: net + key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network') + - prefix: distro + key: cloudscale.image.operating_system +compose: + flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug +strict: False diff --git a/test/integration/targets/inventory_cloudscale/inventory-public.yml b/test/integration/targets/inventory_cloudscale/inventory-public.yml new file mode 100644 index 00000000000..899f66f882e --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/inventory-public.yml @@ -0,0 +1,14 @@ +plugin: cloudscale +ansible_host: public_v4 +inventory_hostname: name +groups: + ansible: inventory_hostname.startswith('ansible') + private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0 +keyed_groups: + - prefix: net + key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network') + - prefix: distro + key: cloudscale.image.operating_system +compose: + flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug +strict: False diff --git a/test/integration/targets/inventory_cloudscale/inventory-uuid.yml b/test/integration/targets/inventory_cloudscale/inventory-uuid.yml new file mode 100644 index 00000000000..8c72e0b34d0 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/inventory-uuid.yml @@ -0,0 +1,14 @@ +plugin: cloudscale +ansible_host: public_v4 +inventory_hostname: uuid +groups: + ansible: cloudscale.name.startswith('ansible') + private_net: (cloudscale.interfaces | selectattr('type', 'equalto', 'private') | list | length) > 0 +keyed_groups: + - prefix: net + key: (cloudscale.interfaces.0.addresses.0.address + '/' + cloudscale.interfaces.0.addresses.0.prefix_length | string) | ipaddr('network') + - prefix: distro + key: cloudscale.image.operating_system +compose: + flavor_image: cloudscale.flavor.slug + '_' + cloudscale.image.slug +strict: False diff --git a/test/integration/targets/inventory_cloudscale/playbooks/change-inventory-config.yml b/test/integration/targets/inventory_cloudscale/playbooks/change-inventory-config.yml new file mode 100644 index 00000000000..74e9132de1f --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/playbooks/change-inventory-config.yml @@ -0,0 +1,8 @@ +- name: Change inventory configuration to {{ inventory_config }} + file: + src: '{{ inventory_config }}' + dest: ../inventory_cloudscale.yml + state: link + +- name: Refresh inventory + meta: refresh_inventory diff --git a/test/integration/targets/inventory_cloudscale/playbooks/cleanup.yml b/test/integration/targets/inventory_cloudscale/playbooks/cleanup.yml new file mode 100644 index 00000000000..acb3c907fcb --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/playbooks/cleanup.yml @@ -0,0 +1,17 @@ +--- +- name: List all servers + uri: + url: 'https://api.cloudscale.ch/v1/servers' + headers: + Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}' + status_code: 200 + register: server_list + +- name: Remove all servers created by this test run + cloudscale_server: + uuid: '{{ item.uuid }}' + state: 'absent' + when: cloudscale_resource_prefix in item.name + with_items: '{{ server_list.json }}' + loop_control: + label: '{{ item.name }} ({{ item.uuid }})' diff --git a/test/integration/targets/inventory_cloudscale/playbooks/common-asserts.yml b/test/integration/targets/inventory_cloudscale/playbooks/common-asserts.yml new file mode 100644 index 00000000000..b83e4ffcf32 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/playbooks/common-asserts.yml @@ -0,0 +1,50 @@ +--- +- name: '{{ inventory }}: Verify basic inventory' + assert: + that: + - server_public[identifier] in hostvars + - server_private[identifier] in hostvars + - server_public_private[identifier] in hostvars + - server_unsafe_chars[identifier] in hostvars + +- name: '{{ inventory }}: Verify duplicate host names in inventory' + assert: + that: + - cloudscale_resource_prefix + '-duplicate' not in hostvars + - (cloudscale_resource_prefix + '-duplicate') | safe_group_name in groups + +- name: '{{ inventory }}: Verify constructed groups in inventory' + assert: + that: + # Test for the "ansible" group + - '"ansible" in groups' + - server_public[identifier] in groups.ansible + - server_private[identifier] in groups.ansible + - server_public_private[identifier] in groups.ansible + - server_unsafe_chars[identifier] in groups.ansible + - server_other_prefix[identifier] not in groups.ansible + # Tests for the "private_net" group + - '"private_net" in groups' + - server_public[identifier] not in groups["private_net"] + - server_private[identifier] in groups["private_net"] + - server_public_private[identifier] in groups["private_net"] + # Tests for "distro" keyed group + - '"distro_Debian" in groups' + - '"distro_Ubuntu" in groups' + - server_public[identifier] in groups.distro_Debian + - server_private[identifier] not in groups.distro_Debian + - server_public[identifier] not in groups.distro_Ubuntu + - server_private[identifier] in groups.distro_Ubuntu + # Test for flavor_image composed variable + - hostvars[server_public[identifier]].flavor_image == 'flex-2_debian-9' + - hostvars[server_private[identifier]].flavor_image == 'flex-2_ubuntu-18.04' + +- name: '{{ inventory }}: Verify cloudscale specific host variables' + assert: + that: + - hostvars[item.0[identifier]].cloudscale[item.1] == item.0[item.1] + with_nested: + - [ '{{ server_public }}', '{{ server_private }}', '{{ server_public_private }}' ] + - [ 'anti_affinity_with', 'flavor', 'href', 'image', 'interfaces', 'name', 'uuid', 'volumes' ] + loop_control: + label: '{{ item.0.name }} ({{ item.0.uuid }}): {{ item.1 }}' diff --git a/test/integration/targets/inventory_cloudscale/playbooks/setup.yml b/test/integration/targets/inventory_cloudscale/playbooks/setup.yml new file mode 100644 index 00000000000..90a57559544 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/playbooks/setup.yml @@ -0,0 +1,74 @@ +--- +- name: Create server with public network only + cloudscale_server: + name: '{{ cloudscale_resource_prefix }}-inventory-public' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_test_image }}' + ssh_keys: '{{ cloudscale_test_ssh_key }}' + use_public_network: True + use_private_network: False + register: server_public + +- name: Create server with private network only + cloudscale_server: + name: '{{ cloudscale_resource_prefix }}-inventory-private' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_alt_test_image }}' + ssh_keys: '{{ cloudscale_test_ssh_key }}' + use_public_network: False + use_private_network: True + register: server_private + +- name: Create server with public and private network + cloudscale_server: + name: '{{ cloudscale_resource_prefix }}-inventory-public-private' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_test_image }}' + ssh_keys: '{{ cloudscale_test_ssh_key }}' + use_public_network: True + use_private_network: True + register: server_public_private + +- name: Create servers with duplicate names + # The cloudscale_server module does not allow creating two servers with the same + # name. To do this the uri module has to be used. + uri: + url: 'https://api.cloudscale.ch/v1/servers' + method: POST + headers: + Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}' + body: + name: '{{ cloudscale_resource_prefix }}-duplicate' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_test_image }}' + ssh_keys: + - '{{ cloudscale_test_ssh_key }}' + body_format: json + status_code: 201 + register: duplicate + with_sequence: count=2 + +- name: Create server with different prefix + cloudscale_server: + name: 'other-prefix-{{ cloudscale_resource_prefix }}-inventory' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_test_image }}' + ssh_keys: '{{ cloudscale_test_ssh_key }}' + register: server_other_prefix + +# The API does not allow creation of a server with a name containing +# characters not allowed in DNS names. So create a server and rename +# it afterwards (which is possible). The resaon for this restriction is +# that on creation a PTR entry for the server is created. +- name: Create server to be renamed with unsafe characters + cloudscale_server: + name: '{{ cloudscale_resource_prefix }}-unsafe-chars' + flavor: '{{ cloudscale_test_flavor }}' + image: '{{ cloudscale_test_image }}' + ssh_keys: '{{ cloudscale_test_ssh_key }}' + register: server_unsafe_chars +- name: Rename server to contain unsafe characters + cloudscale_server: + uuid: '{{ server_unsafe_chars.uuid }}' + name: '{{ cloudscale_resource_prefix }}-snowmans-are-cool-☃!' + register: server_unsafe_chars diff --git a/test/integration/targets/inventory_cloudscale/playbooks/test-inventory.yml b/test/integration/targets/inventory_cloudscale/playbooks/test-inventory.yml new file mode 100644 index 00000000000..24fe38eeb94 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/playbooks/test-inventory.yml @@ -0,0 +1,68 @@ +--- +- name: Create servers and test cloudscale inventory plugin + hosts: localhost + gather_facts: False + roles: + - cloudscale_common + tasks: + - block: + - import_tasks: setup.yml + + - import_tasks: change-inventory-config.yml + vars: + inventory_config: inventory-public.yml + + - import_tasks: common-asserts.yml + vars: + identifier: 'name' + inventory: 'Public v4' + + - name: Verify inventory with public IP + assert: + that: + # Test ansible_host setting + - server_public.interfaces.0.addresses.0.address + == hostvars[server_public.name].ansible_host + - server_public_private.interfaces.0.addresses.0.address + == hostvars[server_public_private.name].ansible_host + - '"ansible_host" not in hostvars[server_private.name]' + + - import_tasks: change-inventory-config.yml + vars: + inventory_config: inventory-private.yml + + - import_tasks: common-asserts.yml + vars: + identifier: 'name' + inventory: 'Private v4' + + - name: Verify inventory with private IP + assert: + that: + # Test ansible_host setting + - '"ansible_host" not in hostvars[server_public.name]' + - server_private.interfaces.0.addresses.0.address + == hostvars[server_private.name].ansible_host + - server_public_private.interfaces.1.addresses.0.address + == hostvars[server_public_private.name].ansible_host + + - import_tasks: change-inventory-config.yml + vars: + inventory_config: inventory-uuid.yml + + - import_tasks: common-asserts.yml + vars: + identifier: 'uuid' + inventory: 'UUID' + + - name: Verify inventory with UUID + assert: + that: + # Test server name groups + - groups[server_public.name | safe_group_name] == [server_public.uuid] + - groups[server_private.name | safe_group_name] == [server_private.uuid] + - groups[server_public_private.name | safe_group_name] == [server_public_private.uuid] + - groups[server_unsafe_chars.name | safe_group_name] == [server_unsafe_chars.uuid] + + always: + - import_tasks: cleanup.yml diff --git a/test/integration/targets/inventory_cloudscale/runme.sh b/test/integration/targets/inventory_cloudscale/runme.sh new file mode 100755 index 00000000000..25c2352d459 --- /dev/null +++ b/test/integration/targets/inventory_cloudscale/runme.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# Exit on errors, exit when accessing unset variables and print all commands +set -eux + +# Set the role path so that the cloudscale_common role is available +export ANSIBLE_ROLES_PATH="../" + +# Set the filter plugin search path so that the safe_group_name filter is available +export ANSIBLE_FILTER_PLUGINS="./filter_plugins" + +rm -f inventory.yml +export ANSIBLE_INVENTORY="./inventory_cloudscale.yml" + +# Run without converting invalid characters in group names +export ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS=never +ansible-playbook playbooks/test-inventory.yml "$@" + +# Run with converting invalid characters in group names +export ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS=always +ansible-playbook playbooks/test-inventory.yml "$@"