diff --git a/lib/ansible/module_utils/azure_rm_common.py b/lib/ansible/module_utils/azure_rm_common.py new file mode 100644 index 00000000000..1c8287648b8 --- /dev/null +++ b/lib/ansible/module_utils/azure_rm_common.py @@ -0,0 +1,619 @@ +# +# Copyright (c) 2016 Matt Davis, +# Chris Houseknecht, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# + +import ConfigParser +import json +import os +import re +import sys +import copy + +from os.path import expanduser +from ansible.module_utils.basic import * + +AZURE_COMMON_ARGS = dict( + profile=dict(type='str'), + subscription_id=dict(type='str', no_log=True), + client_id=dict(type='str', no_log=True), + secret=dict(type='str', no_log=True), + tenant=dict(type='str', no_log=True), +) + +AZURE_CREDENTIAL_ENV_MAPPING = dict( + profile='AZURE_PROFILE', + subscription_id='AZURE_SUBSCRIPTION_ID', + client_id='AZURE_CLIENT_ID', + secret='AZURE_SECRET', + tenant='AZURE_TENANT', + ad_user='AZURE_AD_USER', + password='AZURE_PASSWORD' +) + +AZURE_TAG_ARGS = dict( + tags=dict(type='dict'), + purge_tags=dict(type='bool', default=False), +) + +AZURE_COMMON_REQUIRED_IF = [ + ('log_mode', 'file', ['log_path']) +] + +CIDR_PATTERN = re.compile("(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1" + "[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))") + +AZURE_SUCCESS_STATE = "Succeeded" +AZURE_FAILED_STATE = "Failed" + +AZURE_MIN_VERSION = "2016-03-30" + +HAS_AZURE = True +HAS_AZURE_EXC = None + +try: + from enum import Enum + from msrest.serialization import Serializer + from msrestazure.azure_exceptions import CloudError + from azure.mgmt.compute import __version__ as azure_compute_version + from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, SecurityRule, NetworkInterface, \ + NetworkInterfaceIPConfiguration, Subnet + from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials + from azure.mgmt.network.network_management_client import NetworkManagementClient,\ + NetworkManagementClientConfiguration + from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient,\ + ResourceManagementClientConfiguration + from azure.mgmt.storage.storage_management_client import StorageManagementClient,\ + StorageManagementClientConfiguration + from azure.mgmt.compute.compute_management_client import ComputeManagementClient,\ + ComputeManagementClientConfiguration + from azure.storage.cloudstorageaccount import CloudStorageAccount +except ImportError, exc: + HAS_AZURE_EXC = exc + HAS_AZURE = False + + +def azure_id_to_dict(id): + pieces = re.sub(r'^\/', '', id).split('/') + result = {} + index = 0 + while index < len(pieces) - 1: + result[pieces[index]] = pieces[index + 1] + index += 1 + return result + + +class AzureRMModuleBase(object): + + def __init__(self, derived_arg_spec, bypass_checks=False, no_log=False, + check_invalid_arguments=True, mutually_exclusive=None, required_together=None, + required_one_of=None, add_file_common_args=False, supports_check_mode=False, + required_if=None, supports_tags=True, facts_module=False): + + merged_arg_spec = dict() + merged_arg_spec.update(AZURE_COMMON_ARGS) + if supports_tags: + merged_arg_spec.update(AZURE_TAG_ARGS) + + if derived_arg_spec: + merged_arg_spec.update(derived_arg_spec) + + merged_required_if = list(AZURE_COMMON_REQUIRED_IF) + if required_if: + merged_required_if += required_if + + self.module = AnsibleModule(argument_spec=merged_arg_spec, + bypass_checks=bypass_checks, + no_log=no_log, + check_invalid_arguments=check_invalid_arguments, + mutually_exclusive=mutually_exclusive, + required_together=required_together, + required_one_of=required_one_of, + add_file_common_args=add_file_common_args, + supports_check_mode=supports_check_mode, + required_if=merged_required_if) + + if not HAS_AZURE: + self.fail("The Azure Python SDK is not installed (try 'pip install azure') - {0}".format(HAS_AZURE_EXC)) + + if azure_compute_version < AZURE_MIN_VERSION: + self.fail("Expecting azure.mgmt.compute.__version__ to be >= {0}. Found version {1} " + "Do you have Azure >= 2.0.0rc2 installed?".format(AZURE_MIN_VERSION, azure_compute_version)) + + self._network_client = None + self._storage_client = None + self._resource_client = None + self._compute_client = None + self.check_mode = self.module.check_mode + self.facts_module = facts_module + + # authenticate + self.credentials = self._get_credentials(self.module.params) + if not self.credentials: + self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " + "or define a profile in ~/.azure/credentials.") + + if self.credentials.get('subscription_id', None) is None: + self.fail("Credentials did not include a subscription_id value.") + self.log("setting subscription_id") + self.subscription_id = self.credentials['subscription_id'] + + if self.credentials.get('client_id') is not None and \ + self.credentials.get('secret') is not None and \ + self.credentials.get('tenant') is not None: + self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], + secret=self.credentials['secret'], + tenant=self.credentials['tenant']) + elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + else: + self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " + "Credentials must include client_id, secret and tenant or ad_user and password.") + + # common parameter validation + if self.module.params.get('tags'): + self.validate_tags(self.module.params['tags']) + + res = self.exec_module(**self.module.params) + self.module.exit_json(**res) + + def exec_module(self, **kwargs): + self.fail("Error: {0} failed to implement exec_module method.".format(self.__class__.__name__)) + + def fail(self, msg): + ''' + Shortcut for calling module.fail() + + :param msg: Errot message text. + :return: None + ''' + self.module.fail_json(msg=msg) + + def log(self, msg, pretty_print=False): + pass + # log_file = open('azure_rm.log', 'a') + # if pretty_print: + # log_file.write(json.dumps(msg, indent=4, sort_keys=True)) + # else: + # log_file.write(msg + u'\n') + + def validate_tags(self, tags): + ''' + Check if tags dictionary contains string:string pairs. + + :param tags: dictionary of string:string pairs + :return: None + ''' + if not self.facts_module: + if not isinstance(tags, dict): + self.fail("Tags must be a dictionary of string:string values.") + for key, value in tags.items(): + if not isinstance(value, str): + self.fail("Tags values must be strings. Found {0}:{1}".format(str(key), str(value))) + + def _tag_purge(self, tags): + ''' + Remove metadata tags not found in user provided tags parameter. Returns tuple + with bool indicating something changed and dict of new tags to be assigned to + the object. + + :param tags: object metadata tags + :return: bool, dict of tags + ''' + if not self.module.params.get('tags'): + # purge all tags + return True, dict() + new_tags = copy.copy(tags) + changed = False + for key in tags: + if not self.module.params['tags'].get(key): + # key not found in user provided parameters + new_tags.pop(key) + changed = True + if changed: + self.log('CHANGED: purged tags') + return changed, new_tags + + def _tag_update(self, tags): + ''' + Update metadata tags with values in user provided tags parameter. Returns + tuple with bool indicating something changed and dict of new tags to be + assigned to the object. + + :param tags: object metadata tags + :return: bool, dict of tags + ''' + if isinstance(tags, dict): + new_tags = copy.copy(tags) + else: + new_tags = dict() + changed = False + if self.module.params.get('tags'): + for key, value in self.module.params['tags'].items(): + if not (new_tags.get(key) and new_tags[key] == value): + changed = True + new_tags[key] = value + if changed: + self.log('CHANGED: updated tags') + return changed, new_tags + + def update_tags(self, tags): + ''' + Call from the module to update metadata tags. Returns tuple + with bool indicating if there was a change and dict of new + tags to assign to the object. + + :param tags: metadata tags from the object + :return: bool, dict + ''' + changed = False + updated, new_tags = self._tag_update(tags) + if updated: + changed = True + + if self.module.params['purge_tags']: + purged, new_tags = self._tag_purge(new_tags) + if purged: + changed = True + return changed, new_tags + + def has_tags(self, obj_tags, tag_list): + ''' + Used in fact modules to compare object tags to list of parameter tags. Return true if list of parameter tags + exists in object tags. + + :param obj_tags: dictionary of tags from an Azure object. + :param tag_list: list of tag keys or tag key:value pairs + :return: bool + ''' + + if not obj_tags and tag_list: + return False + + if not tag_list: + return True + + matches = 0 + result = False + for tag in tag_list: + tag_key = tag + tag_value = None + if ':' in tag: + tag_key, tag_value = tag.split(':') + if tag_value and obj_tags.get(tag_key) == tag_value: + matches += 1 + elif not tag_value and obj_tags.get(tag_key): + matches += 1 + if matches == len(tag_list): + result = True + return result + + def get_resource_group(self, resource_group): + ''' + Fetch a resource group. + + :param resource_group: name of a resource group + :return: resource group object + ''' + try: + return self.rm_client.resource_groups.get(resource_group) + except CloudError: + self.fail("Parameter error: resource group {0} not found".format(resource_group)) + except Exception, exc: + self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc))) + + def _get_profile(self, profile="default"): + path = expanduser("~/.azure/credentials") + try: + config = ConfigParser.ConfigParser() + config.read(path) + except Exception, exc: + self.fail("Failed to access {0}. Check that the file exists and you have read " + "access. {1}".format(path, str(exc))) + credentials = dict() + for key in AZURE_CREDENTIAL_ENV_MAPPING: + try: + credentials[key] = config.get(profile, key, raw=True) + except: + pass + + if credentials.get('client_id') is not None or credentials.get('ad_user') is not None: + return credentials + + return None + + def _get_env_credentials(self): + env_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): + env_credentials[attribute] = os.environ.get(env_variable, None) + + if env_credentials['profile'] is not None: + credentials = self._get_profile(env_credentials['profile']) + return credentials + + if env_credentials['client_id'] is not None: + return env_credentials + + return None + + def _get_credentials(self, params): + # Get authentication credentials. + # Precedence: module parameters-> environment variables-> default profile in ~/.azure/credentials. + + self.log('Getting credentials') + + arg_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): + arg_credentials[attribute] = params.get(attribute, None) + + # try module params + if arg_credentials['profile'] is not None: + self.log('Retrieving credentials with profile parameter.') + credentials = self._get_profile(arg_credentials['profile']) + return credentials + + if arg_credentials['client_id'] is not None: + self.log('Received credentials from parameters.') + return arg_credentials + + # try environment + env_credentials = self._get_env_credentials() + if env_credentials: + self.log('Received credentials from env.') + return env_credentials + + # try default profile from ~./azure/credentials + default_credentials = self._get_profile() + if default_credentials: + self.log('Retrieved default profile credentials from ~/.azure/credentials.') + return default_credentials + + return None + + def serialize_obj(self, obj, class_name): + ''' + Return a JSON representation of an Azure object. + + :param obj: Azure object + :param class_name: Name of the object's class + :return: serialized result + ''' + serializer = Serializer() + return serializer.body(obj, class_name) + + def get_poller_result(self, poller): + ''' + Consistent method of waiting on and retrieving results from Azure's long poller + + :param poller Azure poller object + :return object resulting from the original request + ''' + try: + delay = 20 + while not poller.done(): + self.log("Waiting for {0} sec".format(delay)) + poller.wait(timeout=delay) + return poller.result() + except Exception, exc: + self.log(str(exc)) + raise + + def check_provisioning_state(self, azure_object, requested_state='present'): + ''' + Check an Azure object's provisioning state. If something did not complete the provisioning + process, then we cannot operate on it. + + :param azure_object An object such as a subnet, storageaccount, etc. Must have provisioning_state + and name attributes. + :return None + ''' + + if hasattr(azure_object, 'properties') and hasattr(azure_object.properties, 'provisioning_state') and \ + hasattr(azure_object, 'name'): + # resource group object fits this model + if isinstance(azure_object.properties.provisioning_state, Enum): + if azure_object.properties.provisioning_state.value != AZURE_SUCCESS_STATE and \ + requested_state != 'absent': + self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format( + azure_object.name, azure_object.properties.provisioning_state, AZURE_SUCCESS_STATE)) + return + if azure_object.properties.provisioning_state != AZURE_SUCCESS_STATE and \ + requested_state != 'absent': + self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format( + azure_object.name, azure_object.properties.provisioning_state, AZURE_SUCCESS_STATE)) + return + + if hasattr(azure_object, 'provisioning_state') or not hasattr(azure_object, 'name'): + if isinstance(azure_object.provisioning_state, Enum): + if azure_object.provisioning_state.value != AZURE_SUCCESS_STATE and requested_state != 'absent': + self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format( + azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE)) + return + if azure_object.provisioning_state != AZURE_SUCCESS_STATE and requested_state != 'absent': + self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format( + azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE)) + + def get_blob_client(self, resource_group_name, storage_account_name): + keys = dict() + try: + # Get keys from the storage account + self.log('Getting keys') + account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name) + keys['key1'] = account_keys.key1 + keys['key2'] = account_keys.key2 + except Exception, exc: + self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc))) + + try: + self.log('Create blob service') + return CloudStorageAccount(storage_account_name, keys['key1']).create_block_blob_service() + except Exception, exc: + self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name, + str(exc))) + + def create_default_pip(self, resource_group, location, name, allocation_method='Dynamic'): + ''' + Create a default public IP address 01 to associate with a network interface. + If a PIP address matching 01 exists, return it. Otherwise, create one. + + :param resource_group: name of an existing resource group + :param location: a valid azure location + :param name: base name to assign the public IP address + :param allocation_method: one of 'Static' or 'Dynamic' + :return: PIP object + ''' + public_ip_name = name + '01' + pip = None + + self.log("Starting create_default_pip {0}".format(public_ip_name)) + self.log("Check to see if public IP {0} exists".format(public_ip_name)) + try: + pip = self.network_client.public_ip_addresses.get(resource_group, public_ip_name) + except CloudError: + pass + + if pip: + self.log("Public ip {0} found.".format(public_ip_name)) + self.check_provisioning_state(pip) + return pip + + params = PublicIPAddress( + location=location, + public_ip_allocation_method=allocation_method, + ) + self.log('Creating default public IP {0}'.format(public_ip_name)) + try: + poller = self.network_client.public_ip_addresses.create_or_update(resource_group, public_ip_name, params) + except Exception, exc: + self.fail("Error creating {0} - {1}".format(public_ip_name, str(exc))) + + return self.get_poller_result(poller) + + def create_default_securitygroup(self, resource_group, location, name, os_type, open_ports): + ''' + Create a default security group 01 to associate with a network interface. If a security group matching + 01 exists, return it. Otherwise, create one. + + :param resource_group: Resource group name + :param location: azure location name + :param name: base name to use for the security group + :param os_type: one of 'Windows' or 'Linux'. Determins any default rules added to the security group. + :param ssh_port: for os_type 'Linux' port used in rule allowing SSH access. + :param rdp_port: for os_type 'Windows' port used in rule allowing RDP access. + :return: security_group object + ''' + security_group_name = name + '01' + group = None + + self.log("Create security group {0}".format(security_group_name)) + self.log("Check to see if security group {0} exists".format(security_group_name)) + try: + group = self.network_client.network_security_groups.get(resource_group, security_group_name) + except CloudError: + pass + + if group: + self.log("Security group {0} found.".format(security_group_name)) + self.check_provisioning_state(group) + return group + + parameters = NetworkSecurityGroup() + parameters.location = location + + if not open_ports: + # Open default ports based on OS type + if os_type == 'Linux': + # add an inbound SSH rule + parameters.security_rules = [ + SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow SSH Access', + source_port_range='*', destination_port_range='22', priority=100, name='SSH') + ] + parameters.location = location + else: + # for windows add inbound RDP rules + parameters.security_rules = [ + SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow RDP port 3389', + source_port_range='*', destination_port_range='3389', priority=100, name='RDP01'), + SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', description='Allow RDP port 5986', + source_port_range='*', destination_port_range='5986', priority=101, name='RDP01'), + ] + else: + # Open custom ports + parameters.security_rules = [] + priority = 100 + for port in open_ports: + priority += 1 + rule_name = "Rule_{0}".format(priority) + parameters.security_rules.append( + SecurityRule('Tcp', '*', '*', 'Allow', 'Inbound', source_port_range='*', + destination_port_range=str(port), priority=priority, name=rule_name) + ) + + self.log('Creating default security group {0}'.format(security_group_name)) + try: + poller = self.network_client.network_security_groups.create_or_update(resource_group, + security_group_name, + parameters) + except Exception, exc: + self.fail("Error creating default security rule {0} - {1}".format(security_group_name, str(exc))) + + return self.get_poller_result(poller) + + def _register(self, key): + try: + # We have to perform the one-time registration here. Otherwise, we receive an error the first + # time we attempt to use the requested client. + resource_client = self.rm_client + resource_client.providers.register(key) + except Exception, exc: + self.fail("One-time registration of {0} failed - {1}".format(key, str(exc))) + + @property + def storage_client(self): + self.log('Getting storage client...') + if not self._storage_client: + self._storage_client = StorageManagementClient( + StorageManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + self._register('Microsoft.Storage') + return self._storage_client + + @property + def network_client(self): + self.log('Getting network client') + if not self._network_client: + self._network_client = NetworkManagementClient( + NetworkManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + self._register('Microsoft.Network') + return self._network_client + + @property + def rm_client(self): + self.log('Getting resource manager client') + if not self._resource_client: + self._resource_client = ResourceManagementClient( + ResourceManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + return self._resource_client + + @property + def compute_client(self): + self.log('Getting compute client') + if not self._compute_client: + self._compute_client = ComputeManagementClient( + ComputeManagementClientConfiguration(self.azure_credentials, self.subscription_id)) + self._register('Microsoft.Compute') + return self._compute_client