diff --git a/lib/ansible/module_utils/azure_rm_common.py b/lib/ansible/module_utils/azure_rm_common.py index 3b992fd44f9..00886e813ba 100644 --- a/lib/ansible/module_utils/azure_rm_common.py +++ b/lib/ansible/module_utils/azure_rm_common.py @@ -277,7 +277,6 @@ class AzureRMModuleBase(object): self.fail("Do you have azure>={1} installed? Try `pip install ansible[azure]`" "- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE)) - self._cloud_environment = None self._network_client = None self._storage_client = None self._resource_client = None @@ -285,116 +284,23 @@ class AzureRMModuleBase(object): self._dns_client = None self._web_client = None self._marketplace_client = None - self._containerservice_client = None self._sql_client = None self._mysql_client = None self._postgresql_client = None self._containerregistry_client = None self._containerinstance_client = None + self._containerservice_client = None self._traffic_manager_management_client = None self._monitor_client = None - self._adfs_authority_url = None self._resource = None self.check_mode = self.module.check_mode self.api_profile = self.module.params.get('api_profile') self.facts_module = facts_module + # self.debug = self.module.params.get('debug') - # authenticate - self.credentials = self._get_credentials(self.module.params) - if not self.credentials: - if HAS_AZURE_CLI_CORE: - self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " - "define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).") - else: - self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " - "define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).") - - # cert validation mode precedence: module-arg, credential profile, env, "validate" - self._cert_validation_mode = self.module.params['cert_validation_mode'] or self.credentials.get('cert_validation_mode') or \ - os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate' - - if self._cert_validation_mode not in ['validate', 'ignore']: - self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode)) - - # if cloud_environment specified, look up/build Cloud object - raw_cloud_env = self.credentials.get('cloud_environment') - if self.credentials.get('credentials') is not None and raw_cloud_env is not None: - self._cloud_environment = raw_cloud_env - elif not raw_cloud_env: - self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default - else: - # try to look up "well-known" values via the name attribute on azure_cloud members - all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] - matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] - if len(matched_clouds) == 1: - self._cloud_environment = matched_clouds[0] - elif len(matched_clouds) > 1: - self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) - else: - if not urlparse.urlparse(raw_cloud_env).scheme: - self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) - try: - self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) - except Exception as e: - self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e)) - - if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None: - self.fail("Credentials did not include a subscription_id value.") - self.log("setting subscription_id") - self.subscription_id = self.credentials['subscription_id'] - - # get authentication authority - # for adfs, user could pass in authority or not. - # for others, use default authority from cloud environment - if self.credentials.get('adfs_authority_url') is None: - self._adfs_authority_url = self._cloud_environment.endpoints.active_directory - else: - self._adfs_authority_url = self.credentials.get('adfs_authority_url') - - # get resource from cloud environment - self._resource = self._cloud_environment.endpoints.active_directory_resource_id - - if self.credentials.get('credentials') is not None: - # AzureCLI credentials - self.azure_credentials = self.credentials['credentials'] - elif self.credentials.get('client_id') is not None and \ - self.credentials.get('secret') is not None and \ - self.credentials.get('tenant') is not None: - self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], - secret=self.credentials['secret'], - tenant=self.credentials['tenant'], - cloud_environment=self._cloud_environment, - verify=self._cert_validation_mode == 'validate') - - elif self.credentials.get('ad_user') is not None and \ - self.credentials.get('password') is not None and \ - self.credentials.get('client_id') is not None and \ - self.credentials.get('tenant') is not None: - - self.azure_credentials = self.acquire_token_with_username_password( - self._adfs_authority_url, - self._resource, - self.credentials['ad_user'], - self.credentials['password'], - self.credentials['client_id'], - self.credentials['tenant']) - - elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: - tenant = self.credentials.get('tenant') - if not tenant: - tenant = 'common' # SDK default - - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], - self.credentials['password'], - tenant=tenant, - cloud_environment=self._cloud_environment, - verify=self._cert_validation_mode == 'validate') - else: - self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " - "Credentials must include client_id, secret and tenant or ad_user and password, or " - "ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or " - "be logged in using AzureCLI.") + # delegate auth to AzureRMAuth class (shared with all plugin types) + self.azure_auth = AzureRMAuth(fail_impl=self.fail, **self.module.params) # common parameter validation if self.module.params.get('tags'): @@ -404,17 +310,6 @@ class AzureRMModuleBase(object): res = self.exec_module(**self.module.params) self.module.exit_json(**res) - def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant): - authority_uri = authority - - if tenant is not None: - authority_uri = authority + '/' + tenant - - context = AuthenticationContext(authority_uri) - token_response = context.acquire_token_with_username_password(resource, username, password, client_id) - - return AADTokenCredentials(token_response) - def check_client_version(self, client_type): # Ensure Azure modules are at least 2.0.0rc5. package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None) @@ -541,138 +436,6 @@ class AzureRMModuleBase(object): except Exception as exc: self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc))) - def _get_profile(self, profile="default"): - path = expanduser("~/.azure/credentials") - try: - config = configparser.ConfigParser() - config.read(path) - except Exception as exc: - self.fail("Failed to access {0}. Check that the file exists and you have read " - "access. {1}".format(path, str(exc))) - credentials = dict() - for key in AZURE_CREDENTIAL_ENV_MAPPING: - try: - credentials[key] = config.get(profile, key, raw=True) - except: - pass - - if credentials.get('subscription_id'): - return credentials - - return None - - def _get_msi_credentials(self, subscription_id_param=None): - credentials = MSIAuthentication() - subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None) - if not subscription_id: - try: - # use the first subscription of the MSI - subscription_client = SubscriptionClient(credentials) - subscription = next(subscription_client.subscriptions.list()) - subscription_id = str(subscription.subscription_id) - except Exception as exc: - self.fail("Failed to get MSI token: {0}. " - "Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc))) - return { - 'credentials': credentials, - 'subscription_id': subscription_id - } - - def _get_azure_cli_credentials(self): - credentials, subscription_id = get_azure_cli_credentials() - cloud_environment = get_cli_active_cloud() - - cli_credentials = { - 'credentials': credentials, - 'subscription_id': subscription_id, - 'cloud_environment': cloud_environment - } - return cli_credentials - - def _get_env_credentials(self): - env_credentials = dict() - for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): - env_credentials[attribute] = os.environ.get(env_variable, None) - - if env_credentials['profile']: - credentials = self._get_profile(env_credentials['profile']) - return credentials - - if env_credentials.get('subscription_id') is not None: - return env_credentials - - return None - - def _get_credentials(self, params): - # Get authentication credentials. - self.log('Getting credentials') - - arg_credentials = dict() - for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): - arg_credentials[attribute] = params.get(attribute, None) - - auth_source = params.get('auth_source', None) - if not auth_source: - auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto') - - if auth_source == 'msi': - self.log('Retrieving credenitals from MSI') - return self._get_msi_credentials(arg_credentials['subscription_id']) - - if auth_source == 'cli': - if not HAS_AZURE_CLI_CORE: - self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`") - try: - self.log('Retrieving credentials from Azure CLI profile') - cli_credentials = self._get_azure_cli_credentials() - return cli_credentials - except CLIError as err: - self.fail("Azure CLI profile cannot be loaded - {0}".format(err)) - - if auth_source == 'env': - self.log('Retrieving credentials from environment') - env_credentials = self._get_env_credentials() - return env_credentials - - if auth_source == 'credential_file': - self.log("Retrieving credentials from credential file") - profile = params.get('profile', 'default') - default_credentials = self._get_profile(profile) - return default_credentials - - # auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials - # try module params - if arg_credentials['profile'] is not None: - self.log('Retrieving credentials with profile parameter.') - credentials = self._get_profile(arg_credentials['profile']) - return credentials - - if arg_credentials['subscription_id']: - self.log('Received credentials from parameters.') - return arg_credentials - - # try environment - env_credentials = self._get_env_credentials() - if env_credentials: - self.log('Received credentials from env.') - return env_credentials - - # try default profile from ~./azure/credentials - default_credentials = self._get_profile() - if default_credentials: - self.log('Retrieved default profile credentials from ~/.azure/credentials.') - return default_credentials - - try: - if HAS_AZURE_CLI_CORE: - self.log('Retrieving credentials from AzureCLI profile') - cli_credentials = self._get_azure_cli_credentials() - return cli_credentials - except CLIError as ce: - self.log('Error getting AzureCLI profile credentials - {0}'.format(ce)) - - return None - def parse_resource_to_dict(self, resource): ''' Return a dict of the give resource, which contains name and resource group. @@ -947,9 +710,9 @@ class AzureRMModuleBase(object): if not base_url: # most things are resource_manager, don't make everyone specify - base_url = self._cloud_environment.endpoints.resource_manager + base_url = self.azure_auth._cloud_environment.endpoints.resource_manager - client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url) + client_kwargs = dict(credentials=self.azure_auth.azure_credentials, subscription_id=self.azure_auth.subscription_id, base_url=base_url) api_profile_dict = {} @@ -992,11 +755,24 @@ class AzureRMModuleBase(object): if VSCODEEXT_USER_AGENT_KEY in os.environ: client.config.add_user_agent(os.environ[VSCODEEXT_USER_AGENT_KEY]) - if self._cert_validation_mode == 'ignore': + if self.azure_auth._cert_validation_mode == 'ignore': client.config.session_configuration_callback = self._validation_ignore_callback return client + # passthru methods to AzureAuth instance for backcompat + @property + def credentials(self): + return self.azure_auth.credentials + + @property + def _cloud_environment(self): + return self.azure_auth._cloud_environment + + @property + def subscription_id(self): + return self.azure_auth.subscription_id + @property def storage_client(self): self.log('Getting storage client...') @@ -1152,3 +928,281 @@ class AzureRMModuleBase(object): self._monitor_client = self.get_mgmt_svc_client(MonitorManagementClient, base_url=self._cloud_environment.endpoints.resource_manager) return self._monitor_client + + +class AzureRMAuthException(Exception): + pass + + +class AzureRMAuth(object): + def __init__(self, auth_source='auto', profile=None, subscription_id=None, client_id=None, secret=None, + tenant=None, ad_user=None, password=None, cloud_environment='AzureCloud', cert_validation_mode='validate', + api_profile='latest', adfs_authority_url=None, fail_impl=None, **kwargs): + + if fail_impl: + self._fail_impl = fail_impl + else: + self._fail_impl = self._default_fail_impl + + self._cloud_environment = None + self._adfs_authority_url = None + + # authenticate + self.credentials = self._get_credentials( + dict(auth_source=auth_source, profile=profile, subscription_id=subscription_id, client_id=client_id, secret=secret, + tenant=tenant, ad_user=ad_user, password=password, cloud_environment=cloud_environment, + cert_validation_mode=cert_validation_mode, api_profile=api_profile, adfs_authority_url=adfs_authority_url)) + + if not self.credentials: + if HAS_AZURE_CLI_CORE: + self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " + "define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).") + else: + self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " + "define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).") + + # cert validation mode precedence: module-arg, credential profile, env, "validate" + self._cert_validation_mode = cert_validation_mode or self.credentials.get('cert_validation_mode') or \ + os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate' + + if self._cert_validation_mode not in ['validate', 'ignore']: + self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode)) + + # if cloud_environment specified, look up/build Cloud object + raw_cloud_env = self.credentials.get('cloud_environment') + if self.credentials.get('credentials') is not None and raw_cloud_env is not None: + self._cloud_environment = raw_cloud_env + elif not raw_cloud_env: + self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default + else: + # try to look up "well-known" values via the name attribute on azure_cloud members + all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] + matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] + if len(matched_clouds) == 1: + self._cloud_environment = matched_clouds[0] + elif len(matched_clouds) > 1: + self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) + else: + if not urlparse.urlparse(raw_cloud_env).scheme: + self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) + try: + self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) + except Exception as e: + self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e)) + + if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None: + self.fail("Credentials did not include a subscription_id value.") + self.log("setting subscription_id") + self.subscription_id = self.credentials['subscription_id'] + + # get authentication authority + # for adfs, user could pass in authority or not. + # for others, use default authority from cloud environment + if self.credentials.get('adfs_authority_url') is None: + self._adfs_authority_url = self._cloud_environment.endpoints.active_directory + else: + self._adfs_authority_url = self.credentials.get('adfs_authority_url') + + # get resource from cloud environment + self._resource = self._cloud_environment.endpoints.active_directory_resource_id + + if self.credentials.get('credentials') is not None: + # AzureCLI credentials + self.azure_credentials = self.credentials['credentials'] + elif self.credentials.get('client_id') is not None and \ + self.credentials.get('secret') is not None and \ + self.credentials.get('tenant') is not None: + self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], + secret=self.credentials['secret'], + tenant=self.credentials['tenant'], + cloud_environment=self._cloud_environment, + verify=self._cert_validation_mode == 'validate') + + elif self.credentials.get('ad_user') is not None and \ + self.credentials.get('password') is not None and \ + self.credentials.get('client_id') is not None and \ + self.credentials.get('tenant') is not None: + + self.azure_credentials = self.acquire_token_with_username_password( + self._adfs_authority_url, + self._resource, + self.credentials['ad_user'], + self.credentials['password'], + self.credentials['client_id'], + self.credentials['tenant']) + + elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: + tenant = self.credentials.get('tenant') + if not tenant: + tenant = 'common' # SDK default + + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], + self.credentials['password'], + tenant=tenant, + cloud_environment=self._cloud_environment, + verify=self._cert_validation_mode == 'validate') + else: + self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " + "Credentials must include client_id, secret and tenant or ad_user and password, or " + "ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or " + "be logged in using AzureCLI.") + + def fail(self, msg, exception=None, **kwargs): + self._fail_impl(msg) + + def _default_fail_impl(self, msg, exception=None, **kwargs): + raise AzureRMAuthException(msg) + + def _get_profile(self, profile="default"): + path = expanduser("~/.azure/credentials") + try: + config = configparser.ConfigParser() + config.read(path) + except Exception as exc: + self.fail("Failed to access {0}. Check that the file exists and you have read " + "access. {1}".format(path, str(exc))) + credentials = dict() + for key in AZURE_CREDENTIAL_ENV_MAPPING: + try: + credentials[key] = config.get(profile, key, raw=True) + except: + pass + + if credentials.get('subscription_id'): + return credentials + + return None + + def _get_msi_credentials(self, subscription_id_param=None): + credentials = MSIAuthentication() + subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None) + if not subscription_id: + try: + # use the first subscription of the MSI + subscription_client = SubscriptionClient(credentials) + subscription = next(subscription_client.subscriptions.list()) + subscription_id = str(subscription.subscription_id) + except Exception as exc: + self.fail("Failed to get MSI token: {0}. " + "Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc))) + return { + 'credentials': credentials, + 'subscription_id': subscription_id + } + + def _get_azure_cli_credentials(self): + credentials, subscription_id = get_azure_cli_credentials() + cloud_environment = get_cli_active_cloud() + + cli_credentials = { + 'credentials': credentials, + 'subscription_id': subscription_id, + 'cloud_environment': cloud_environment + } + return cli_credentials + + def _get_env_credentials(self): + env_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): + env_credentials[attribute] = os.environ.get(env_variable, None) + + if env_credentials['profile']: + credentials = self._get_profile(env_credentials['profile']) + return credentials + + if env_credentials.get('subscription_id') is not None: + return env_credentials + + return None + + # TODO: use explicit kwargs instead of intermediate dict + def _get_credentials(self, params): + # Get authentication credentials. + self.log('Getting credentials') + + arg_credentials = dict() + for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items(): + arg_credentials[attribute] = params.get(attribute, None) + + auth_source = params.get('auth_source', None) + if not auth_source: + auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto') + + if auth_source == 'msi': + self.log('Retrieving credenitals from MSI') + return self._get_msi_credentials(arg_credentials['subscription_id']) + + if auth_source == 'cli': + if not HAS_AZURE_CLI_CORE: + self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`") + try: + self.log('Retrieving credentials from Azure CLI profile') + cli_credentials = self._get_azure_cli_credentials() + return cli_credentials + except CLIError as err: + self.fail("Azure CLI profile cannot be loaded - {0}".format(err)) + + if auth_source == 'env': + self.log('Retrieving credentials from environment') + env_credentials = self._get_env_credentials() + return env_credentials + + if auth_source == 'credential_file': + self.log("Retrieving credentials from credential file") + profile = params.get('profile', 'default') + default_credentials = self._get_profile(profile) + return default_credentials + + # auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials + # try module params + if arg_credentials['profile'] is not None: + self.log('Retrieving credentials with profile parameter.') + credentials = self._get_profile(arg_credentials['profile']) + return credentials + + if arg_credentials['subscription_id']: + self.log('Received credentials from parameters.') + return arg_credentials + + # try environment + env_credentials = self._get_env_credentials() + if env_credentials: + self.log('Received credentials from env.') + return env_credentials + + # try default profile from ~./azure/credentials + default_credentials = self._get_profile() + if default_credentials: + self.log('Retrieved default profile credentials from ~/.azure/credentials.') + return default_credentials + + try: + if HAS_AZURE_CLI_CORE: + self.log('Retrieving credentials from AzureCLI profile') + cli_credentials = self._get_azure_cli_credentials() + return cli_credentials + except CLIError as ce: + self.log('Error getting AzureCLI profile credentials - {0}'.format(ce)) + + return None + + def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant): + authority_uri = authority + + if tenant is not None: + authority_uri = authority + '/' + tenant + + context = AuthenticationContext(authority_uri) + token_response = context.acquire_token_with_username_password(resource, username, password, client_id) + + return AADTokenCredentials(token_response) + + def log(self, msg, pretty_print=False): + pass + # Use only during module development + # if self.debug: + # log_file = open('azure_rm.log', 'a') + # if pretty_print: + # log_file.write(json.dumps(msg, indent=4, sort_keys=True)) + # else: + # log_file.write(msg + u'\n') diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 8244b587476..77159eaaecd 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -286,7 +286,7 @@ class Constructable(object): composite = self._compose(compose[varname], variables) except Exception as e: if strict: - raise AnsibleError("Could not set %s: %s" % (varname, to_native(e))) + raise AnsibleError("Could not set %s for host %s: %s" % (varname, host, to_native(e))) continue self.inventory.set_variable(host, varname, composite) diff --git a/lib/ansible/plugins/inventory/azure_rm.py b/lib/ansible/plugins/inventory/azure_rm.py new file mode 100644 index 00000000000..e1c4846f0c4 --- /dev/null +++ b/lib/ansible/plugins/inventory/azure_rm.py @@ -0,0 +1,540 @@ +# Copyright (c) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = r''' + name: azure_rm + plugin_type: inventory + short_description: Azure Resource Manager inventory plugin + extends_documentation_fragment: + - azure + description: + - Query VM details from Azure Resource Manager + - Requires a YAML configuration file whose name ends with '.azure_rm.yaml' + - By default, sets C(ansible_host) to the first public IP address found (preferring the primary NIC). If no + public IPs are found, the first private IP (also preferring the primary NIC). The default may be overridden + via C(hostvar_expressions); see examples. + options: + plugin: + description: marks this as an instance of the 'azure_rm' plugin + required: true + choices: ['azure_rm'] + include_vm_resource_groups: + description: A list of resource group names to search for virtual machines. '\*' will include all resource + groups in the subscription. + default: ['*'] + include_vmss_resource_groups: + description: A list of resource group names to search for virtual machine scale sets (VMSSs). '\*' will + include all resource groups in the subscription. + default: [] + fail_on_template_errors: + description: When false, template failures during group and filter processing are silently ignored (eg, + if a filter or group expression refers to an undefined host variable) + choices: [True, False] + default: True + keyed_groups: + description: Creates groups based on the value of a host variable. Requires a list of dictionaries, + defining C(key) (the source dictionary-typed variable), C(prefix) (the prefix to use for the new group + name), and optionally C(separator) (which defaults to C(_)) + conditional_groups: + description: A mapping of group names to Jinja2 expressions. When the mapped expression is true, the host + is added to the named group. + hostvar_expressions: + description: A mapping of hostvar names to Jinja2 expressions. The value for each host is the result of the + Jinja2 expression (which may refer to any of the host's existing variables at the time this inventory + plugin runs). + exclude_host_filters: + description: Excludes hosts from the inventory with a list of Jinja2 conditional expressions. Each + expression in the list is evaluated for each host; when the expression is true, the host is excluded + from the inventory. + default: [] + batch_fetch: + description: To improve performance, results are fetched using an unsupported batch API. Disabling + C(batch_fetch) uses a much slower serial fetch, resulting in many more round-trips. Generally only + useful for troubleshooting. + default: true + default_host_filters: + description: A default set of filters that is applied in addition to the conditions in + C(exclude_host_filters) to exclude powered-off and not-fully-provisioned hosts. Set this to a different + value or empty list if you need to include hosts in these states. + default: ['powerstate != "running"', 'provisioning_state != "succeeded"'] +''' + +EXAMPLES = ''' +# The following host variables are always available: +# public_ipv4_addresses: all public IP addresses, with the primary IP config from the primary NIC first +# public_dns_hostnames: all public DNS hostnames, with the primary IP config from the primary NIC first +# private_ipv4_addresses: all private IP addressses, with the primary IP config from the primary NIC first +# id: the VM's Azure resource ID, eg /subscriptions/00000000-0000-0000-1111-1111aaaabb/resourceGroups/my_rg/providers/Microsoft.Compute/virtualMachines/my_vm +# location: the VM's Azure location, eg 'westus', 'eastus' +# name: the VM's resource name, eg 'myvm' +# powerstate: the VM's current power state, eg: 'running', 'stopped', 'deallocated' +# provisioning_state: the VM's current provisioning state, eg: 'succeeded' +# tags: dictionary of the VM's defined tag values +# resource_type: the VM's resource type, eg: 'Microsoft.Compute/virtualMachine', 'Microsoft.Compute/virtualMachineScaleSets/virtualMachines' +# vmid: the VM's internal SMBIOS ID, eg: '36bca69d-c365-4584-8c06-a62f4a1dc5d2' +# vmss: if the VM is a member of a scaleset (vmss), a dictionary including the id and name of the parent scaleset + + +# sample 'myazuresub.azure_rm.yaml' + +# required for all azure_rm inventory plugin configs +plugin: azure_rm + +# forces this plugin to use a CLI auth session instead of the automatic auth source selection (eg, prevents the +# presence of 'ANSIBLE_AZURE_RM_X' environment variables from overriding CLI auth) +auth_source: cli + +# fetches VMs from an explicit list of resource groups instead of default all (- '*') +include_vm_resource_groups: +- myrg1 +- myrg2 + +# fetches VMs from VMSSs in all resource groups (defaults to no VMSS fetch) +include_vmss_resource_groups: +- '*' + +# places a host in the named group if the associated condition evaluates to true +conditional_groups: + # since this will be true for every host, every host sourced from this inventory plugin config will be in the + # group 'all_the_hosts' + all_the_hosts: true + # if the VM's "name" variable contains "dbserver", it will be placed in the 'db_hosts' group + db_hosts: "'dbserver' in name" + +# adds variables to each host found by this inventory plugin, whose values are the result of the associated expression +hostvar_expressions: + my_host_var: + # A statically-valued expression has to be both single and double-quoted, or use escaped quotes, since the outer + # layer of quotes will be consumed by YAML. Without the second set of quotes, it interprets 'staticvalue' as a + # variable instead of a string literal. + some_statically_valued_var: "'staticvalue'" + # overrides the default ansible_host value with a custom Jinja2 expression, in this case, the first DNS hostname, or + # if none are found, the first public IP address. + ansible_host: (public_dns_hostnames + public_ipv4_addresses) | first + +# places hosts in dynamically-created groups based on a variable value. +keyed_groups: +# places each host in a group named 'tag_(tag name)_(tag value)' for each tag on a VM. +- prefix: tag + key: tags +# places each host in a group named 'azure_loc_(location name)', depending on the VM's location +- prefix: azure_loc + key: location +# places host in a group named 'some_tag_X' using the value of the 'sometag' tag on a VM as X, and defaulting to the +# value 'none' (eg, the group 'some_tag_none') if the 'sometag' tag is not defined for a VM. +- prefix: some_tag + key: tags.sometag | default('none') + +# excludes a host from the inventory when any of these expressions is true, can refer to any vars defined on the host +exclude_host_filters: +# excludes hosts in the eastus region +- location in ['eastus'] +# excludes hosts that are powered off +- powerstate != 'running' +''' + +# FUTURE: do we need a set of sane default filters, separate from the user-defineable ones? +# eg, powerstate==running, provisioning_state==succeeded + + +import hashlib +import json +import re + +try: + from queue import Queue, Empty +except ImportError: + from Queue import Queue, Empty + +from collections import namedtuple +from ansible import release +from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable +from ansible.module_utils.six import iteritems +from ansible.module_utils.azure_rm_common import AzureRMAuth +from ansible.errors import AnsibleParserError, AnsibleError +from ansible.module_utils.parsing.convert_bool import boolean +from ansible.module_utils._text import to_native +from itertools import chain +from msrest import ServiceClient, Serializer, Deserializer +from msrestazure import AzureConfiguration +from msrestazure.polling.arm_polling import ARMPolling + + +class AzureRMRestConfiguration(AzureConfiguration): + def __init__(self, credentials, subscription_id, base_url=None): + + if credentials is None: + raise ValueError("Parameter 'credentials' must not be None.") + if subscription_id is None: + raise ValueError("Parameter 'subscription_id' must not be None.") + if not base_url: + base_url = 'https://management.azure.com' + + super(AzureRMRestConfiguration, self).__init__(base_url) + + self.add_user_agent('ansible-dynamic-inventory/{0}'.format(release.__version__)) + + self.credentials = credentials + self.subscription_id = subscription_id + + +UrlAction = namedtuple('UrlAction', ['url', 'api_version', 'handler', 'handler_args']) + + +# FUTURE: add Cacheable support once we have a sane serialization format +class InventoryModule(BaseInventoryPlugin, Constructable): + + NAME = 'azure_rm' + + def __init__(self): + super(InventoryModule, self).__init__() + + self._serializer = Serializer() + self._deserializer = Deserializer() + self._hosts = [] + self._filters = None + + # FUTURE: use API profiles with defaults + self._compute_api_version = '2017-03-30' + self._network_api_version = '2015-06-15' + + self._default_header_parameters = {'Content-Type': 'application/json; charset=utf-8'} + + self._request_queue = Queue() + + self.azure_auth = None + + self._batch_fetch = False + + def verify_file(self, path): + ''' + :param loader: an ansible.parsing.dataloader.DataLoader object + :param path: the path to the inventory config file + :return the contents of the config file + ''' + if super(InventoryModule, self).verify_file(path): + if re.match(r'.+\.azure_rm\.y(a)?ml$', path): + return True + # display.debug("azure_rm inventory filename must match '*.azure_rm.yml' or '*.azure_rm.yaml'") + return False + + def parse(self, inventory, loader, path, cache=True): + super(InventoryModule, self).parse(inventory, loader, path) + + self._read_config_data(path) + self._batch_fetch = self.get_option('batch_fetch') + + self._filters = self.get_option('exclude_host_filters') + self.get_option('default_host_filters') + + try: + self._credential_setup() + self._get_hosts() + except Exception as ex: + raise + + def _credential_setup(self): + auth_options = dict( + auth_source=self.get_option('auth_source'), + profile=self.get_option('profile'), + subscription_id=self.get_option('subscription_id'), + client_id=self.get_option('client_id'), + secret=self.get_option('secret'), + tenant=self.get_option('tenant'), + ad_user=self.get_option('ad_user'), + password=self.get_option('password'), + cloud_environment=self.get_option('cloud_environment'), + cert_validation_mode=self.get_option('cert_validation_mode'), + api_profile=self.get_option('api_profile'), + adfs_authority_url=self.get_option('adfs_authority_url') + ) + + self.azure_auth = AzureRMAuth(**auth_options) + + self._clientconfig = AzureRMRestConfiguration(self.azure_auth.azure_credentials, self.azure_auth.subscription_id, + self.azure_auth._cloud_environment.endpoints.resource_manager) + self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig) + + def _enqueue_get(self, url, api_version, handler, handler_args=None): + if not handler_args: + handler_args = {} + self._request_queue.put_nowait(UrlAction(url=url, api_version=api_version, handler=handler, handler_args=handler_args)) + + def _enqueue_vm_list(self, rg='*'): + if not rg or rg == '*': + url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines' + else: + url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines' + + url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) + self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response) + + def _enqueue_vmss_list(self, rg=None): + if not rg or rg == '*': + url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets' + else: + url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets' + + url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg) + self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vmss_page_response) + + def _get_hosts(self): + for vm_rg in self.get_option('include_vm_resource_groups'): + self._enqueue_vm_list(vm_rg) + + for vmss_rg in self.get_option('include_vmss_resource_groups'): + self._enqueue_vmss_list(vmss_rg) + + if self._batch_fetch: + self._process_queue_batch() + else: + self._process_queue_serial() + + constructable_config_strict = boolean(self.get_option('fail_on_template_errors')) + constructable_config_compose = self.get_option('hostvar_expressions') + constructable_config_groups = self.get_option('conditional_groups') + constructable_config_keyed_groups = self.get_option('keyed_groups') + + for h in self._hosts: + inventory_hostname = self._get_hostname(h) + if self._filter_host(inventory_hostname, h.hostvars): + continue + self.inventory.add_host(inventory_hostname) + # FUTURE: configurable default IP list? can already do this via hostvar_expressions + self.inventory.set_variable(inventory_hostname, "ansible_host", + next(chain(h.hostvars['public_ipv4_addresses'], h.hostvars['private_ipv4_addresses']), None)) + for k, v in iteritems(h.hostvars): + # FUTURE: configurable hostvar prefix? Makes docs harder... + self.inventory.set_variable(inventory_hostname, k, v) + + # constructable delegation + self._set_composite_vars(constructable_config_compose, h.hostvars, inventory_hostname, strict=constructable_config_strict) + self._add_host_to_composed_groups(constructable_config_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) + self._add_host_to_keyed_groups(constructable_config_keyed_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict) + + # FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host + def _filter_host(self, inventory_hostname, hostvars): + self.templar.set_available_variables(hostvars) + + for condition in self._filters: + # FUTURE: should warn/fail if conditional doesn't return True or False + conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format(condition) + try: + if boolean(self.templar.template(conditional)): + return True + except Exception as e: + if boolean(self.get_option('fail_on_template_errors')): + raise AnsibleParserError("Error evaluating filter condition '{0}' for host {1}: {2}".format(condition, inventory_hostname, to_native(e))) + continue + + return False + + def _get_hostname(self, host): + # FUTURE: configurable hostname sources + return host.default_inventory_hostname + + def _process_queue_serial(self): + try: + while True: + item = self._request_queue.get_nowait() + resp = self.send_request(item.url, item.api_version) + item.handler(resp, **item.handler_args) + except Empty: + pass + + def _on_vm_page_response(self, response, vmss=None): + next_link = response.get('nextLink') + + if next_link: + self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vm_page_response) + + for h in response['value']: + # FUTURE: add direct VM filtering by tag here (performance optimization)? + self._hosts.append(AzureHost(h, self, vmss=vmss)) + + def _on_vmss_page_response(self, response): + next_link = response.get('nextLink') + + if next_link: + self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vmss_page_response) + + # FUTURE: add direct VMSS filtering by tag here (performance optimization)? + for vmss in response['value']: + url = '{0}/virtualMachines'.format(vmss['id']) + # VMSS instances look close enough to regular VMs that we can share the handler impl... + self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response, handler_args=dict(vmss=vmss)) + + # use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip + # + def _process_queue_batch(self): + while True: + batch_requests = [] + batch_item_index = 0 + batch_response_handlers = [] + try: + while batch_item_index < 500: + item = self._request_queue.get_nowait() + + query_parameters = {'api-version': item.api_version} + req = self._client.get(item.url, query_parameters) + + batch_requests.append(dict(httpMethod="GET", url=req.url)) + batch_response_handlers.append(item) + batch_item_index += 1 + except Empty: + pass + + if not batch_requests: + break + + batch_resp = self._send_batch(batch_requests) + + for idx, r in enumerate(batch_resp['responses']): + status_code = r.get('httpStatusCode') + if status_code != 200: + # FUTURE: error-tolerant operation mode (eg, permissions) + raise AnsibleError("a batched request failed with status code {0}, url {1}".format(status_code, batch_requests[idx].get('url'))) + + item = batch_response_handlers[idx] + # FUTURE: store/handle errors from individual handlers + item.handler(r['content'], **item.handler_args) + + def _send_batch(self, batched_requests): + url = '/batch' + query_parameters = {'api-version': '2015-11-01'} + + body_obj = dict(requests=batched_requests) + + body_content = self._serializer.body(body_obj, 'object') + + request = self._client.post(url, query_parameters) + initial_response = self._client.send(request, self._default_header_parameters, body_content) + + # FUTURE: configurable timeout? + poller = ARMPolling(timeout=2) + poller.initialize(client=self._client, + initial_response=initial_response, + deserialization_callback=lambda r: self._deserializer('object', r)) + + poller.run() + + return poller.resource() + + def send_request(self, url, api_version): + query_parameters = {'api-version': api_version} + req = self._client.get(url, query_parameters) + resp = self._client.send(req, self._default_header_parameters, stream=False) + + resp.raise_for_status() + content = resp.content + + return json.loads(content) + + +# VM list (all, N resource groups): VM -> InstanceView, N NICs, N PublicIPAddress) +# VMSS VMs (all SS, N specific SS, N resource groups?): SS -> VM -> InstanceView, N NICs, N PublicIPAddress) + +class AzureHost(object): + _powerstate_regex = re.compile('^PowerState/(?P.+)$') + + def __init__(self, vm_model, inventory_client, vmss=None): + self._inventory_client = inventory_client + self._vm_model = vm_model + self._vmss = vmss + + self._instanceview = None + + self._powerstate = "unknown" + self.nics = [] + + # Azure often doesn't provide a globally-unique filename, so use resource name + a chunk of ID hash + self.default_inventory_hostname = '{0}_{1}'.format(vm_model['name'], hashlib.sha1(vm_model['id']).hexdigest()[0:4]) + + self._hostvars = {} + + inventory_client._enqueue_get(url="{0}/instanceView".format(vm_model['id']), + api_version=self._inventory_client._compute_api_version, + handler=self._on_instanceview_response) + + nic_refs = vm_model['properties']['networkProfile']['networkInterfaces'] + for nic in nic_refs: + # single-nic instances don't set primary, so figure it out... + is_primary = nic.get('properties', {}).get('primary', len(nic_refs) == 1) + inventory_client._enqueue_get(url=nic['id'], api_version=self._inventory_client._network_api_version, + handler=self._on_nic_response, + handler_args=dict(is_primary=is_primary)) + + @property + def hostvars(self): + if self._hostvars != {}: + return self._hostvars + + new_hostvars = dict( + public_ipv4_addresses=[], + public_dns_hostnames=[], + private_ipv4_addresses=[], + id=self._vm_model['id'], + location=self._vm_model['location'], + name=self._vm_model['name'], + powerstate=self._powerstate, + provisioning_state=self._vm_model['properties']['provisioningState'].lower(), + tags=self._vm_model.get('tags', {}), + resource_type=self._vm_model.get('type', "unknown"), + vmid=self._vm_model['properties']['vmId'], + vmss=dict( + id=self._vmss['id'], + name=self._vmss['name'], + ) if self._vmss else {} + ) + + # set nic-related values from the primary NIC first + for nic in sorted(self.nics, key=lambda n: n.is_primary, reverse=True): + # and from the primary IP config per NIC first + for ipc in sorted(nic._nic_model['properties']['ipConfigurations'], key=lambda i: i['properties']['primary'], reverse=True): + private_ip = ipc['properties'].get('privateIPAddress') + if private_ip: + new_hostvars['private_ipv4_addresses'].append(private_ip) + pip_id = ipc['properties'].get('publicIPAddress', {}).get('id') + if pip_id: + pip = nic.public_ips[pip_id] + new_hostvars['public_ipv4_addresses'].append(pip._pip_model['properties']['ipAddress']) + pip_fqdn = pip._pip_model['properties'].get('dnsSettings', {}).get('fqdn') + if pip_fqdn: + new_hostvars['public_dns_hostnames'].append(pip_fqdn) + + self._hostvars = new_hostvars + + return self._hostvars + + def _on_instanceview_response(self, vm_instanceview_model): + self._instanceview = vm_instanceview_model + self._powerstate = next((self._powerstate_regex.match(s.get('code', '')).group('powerstate') + for s in vm_instanceview_model.get('statuses', []) if self._powerstate_regex.match(s.get('code', ''))), 'unknown') + + def _on_nic_response(self, nic_model, is_primary=False): + nic = AzureNic(nic_model=nic_model, inventory_client=self._inventory_client, is_primary=is_primary) + self.nics.append(nic) + + +class AzureNic(object): + def __init__(self, nic_model, inventory_client, is_primary=False): + self._nic_model = nic_model + self.is_primary = is_primary + self._inventory_client = inventory_client + + self.public_ips = {} + + for ipc in nic_model['properties']['ipConfigurations']: + pip = ipc['properties'].get('publicIPAddress') + if pip: + self._inventory_client._enqueue_get(url=pip['id'], api_version=self._inventory_client._network_api_version, handler=self._on_pip_response) + + def _on_pip_response(self, pip_model): + self.public_ips[pip_model['id']] = AzurePip(pip_model) + + +class AzurePip(object): + def __init__(self, pip_model): + self._pip_model = pip_model