initial commit of azure_rm inventory plugin (#44944)

* crusty refactor of azure_rm to support auth from non-modules

lib/ansible/module_utils/azure_rm_common.py

@@ -277,7 +277,6 @@ class AzureRMModuleBase(object):
             self.fail("Do you have azure>={1} installed? Try `pip install ansible[azure]`"
                       "- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
 
-        self._cloud_environment = None
         self._network_client = None
         self._storage_client = None
         self._resource_client = None
@@ -285,116 +284,23 @@ class AzureRMModuleBase(object):
         self._dns_client = None
         self._web_client = None
         self._marketplace_client = None
-        self._containerservice_client = None
         self._sql_client = None
         self._mysql_client = None
         self._postgresql_client = None
         self._containerregistry_client = None
         self._containerinstance_client = None
+        self._containerservice_client = None
         self._traffic_manager_management_client = None
         self._monitor_client = None
-        self._adfs_authority_url = None
         self._resource = None
 
         self.check_mode = self.module.check_mode
         self.api_profile = self.module.params.get('api_profile')
         self.facts_module = facts_module
+        # self.debug = self.module.params.get('debug')
 
-        # authenticate
+        # delegate auth to AzureRMAuth class (shared with all plugin types)
-        self.credentials = self._get_credentials(self.module.params)
+        self.azure_auth = AzureRMAuth(fail_impl=self.fail, **self.module.params)
-        if not self.credentials:
-            if HAS_AZURE_CLI_CORE:
-                self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
-                          "define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).")
-            else:
-                self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
-                          "define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).")
-
-        # cert validation mode precedence: module-arg, credential profile, env, "validate"
-        self._cert_validation_mode = self.module.params['cert_validation_mode'] or self.credentials.get('cert_validation_mode') or \
-            os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate'
-
-        if self._cert_validation_mode not in ['validate', 'ignore']:
-            self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode))
-
-        # if cloud_environment specified, look up/build Cloud object
-        raw_cloud_env = self.credentials.get('cloud_environment')
-        if self.credentials.get('credentials') is not None and raw_cloud_env is not None:
-            self._cloud_environment = raw_cloud_env
-        elif not raw_cloud_env:
-            self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD  # SDK default
-        else:
-            # try to look up "well-known" values via the name attribute on azure_cloud members
-            all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
-            matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
-            if len(matched_clouds) == 1:
-                self._cloud_environment = matched_clouds[0]
-            elif len(matched_clouds) > 1:
-                self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
-            else:
-                if not urlparse.urlparse(raw_cloud_env).scheme:
-                    self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
-                try:
-                    self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
-                except Exception as e:
-                    self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
-
-        if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None:
-            self.fail("Credentials did not include a subscription_id value.")
-        self.log("setting subscription_id")
-        self.subscription_id = self.credentials['subscription_id']
-
-        # get authentication authority
-        # for adfs, user could pass in authority or not.
-        # for others, use default authority from cloud environment
-        if self.credentials.get('adfs_authority_url') is None:
-            self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
-        else:
-            self._adfs_authority_url = self.credentials.get('adfs_authority_url')
-
-        # get resource from cloud environment
-        self._resource = self._cloud_environment.endpoints.active_directory_resource_id
-
-        if self.credentials.get('credentials') is not None:
-            # AzureCLI credentials
-            self.azure_credentials = self.credentials['credentials']
-        elif self.credentials.get('client_id') is not None and \
-                self.credentials.get('secret') is not None and \
-                self.credentials.get('tenant') is not None:
-                self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
-                                                                     secret=self.credentials['secret'],
-                                                                     tenant=self.credentials['tenant'],
-                                                                     cloud_environment=self._cloud_environment,
-                                                                     verify=self._cert_validation_mode == 'validate')
-
-        elif self.credentials.get('ad_user') is not None and \
-                self.credentials.get('password') is not None and \
-                self.credentials.get('client_id') is not None and \
-                self.credentials.get('tenant') is not None:
-
-                self.azure_credentials = self.acquire_token_with_username_password(
-                    self._adfs_authority_url,
-                    self._resource,
-                    self.credentials['ad_user'],
-                    self.credentials['password'],
-                    self.credentials['client_id'],
-                    self.credentials['tenant'])
-
-        elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
-            tenant = self.credentials.get('tenant')
-            if not tenant:
-                tenant = 'common'  # SDK default
-
-            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
-                                                         self.credentials['password'],
-                                                         tenant=tenant,
-                                                         cloud_environment=self._cloud_environment,
-                                                         verify=self._cert_validation_mode == 'validate')
-        else:
-            self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
-                      "Credentials must include client_id, secret and tenant or ad_user and password, or "
-                      "ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or "
-                      "be logged in using AzureCLI.")
 
         # common parameter validation
         if self.module.params.get('tags'):
@@ -404,17 +310,6 @@ class AzureRMModuleBase(object):
             res = self.exec_module(**self.module.params)
             self.module.exit_json(**res)
 
-    def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant):
-        authority_uri = authority
-
-        if tenant is not None:
-            authority_uri = authority + '/' + tenant
-
-        context = AuthenticationContext(authority_uri)
-        token_response = context.acquire_token_with_username_password(resource, username, password, client_id)
-
-        return AADTokenCredentials(token_response)
-
     def check_client_version(self, client_type):
         # Ensure Azure modules are at least 2.0.0rc5.
         package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None)
@@ -541,138 +436,6 @@ class AzureRMModuleBase(object):
         except Exception as exc:
             self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc)))
 
-    def _get_profile(self, profile="default"):
-        path = expanduser("~/.azure/credentials")
-        try:
-            config = configparser.ConfigParser()
-            config.read(path)
-        except Exception as exc:
-            self.fail("Failed to access {0}. Check that the file exists and you have read "
-                      "access. {1}".format(path, str(exc)))
-        credentials = dict()
-        for key in AZURE_CREDENTIAL_ENV_MAPPING:
-            try:
-                credentials[key] = config.get(profile, key, raw=True)
-            except:
-                pass
-
-        if credentials.get('subscription_id'):
-            return credentials
-
-        return None
-
-    def _get_msi_credentials(self, subscription_id_param=None):
-        credentials = MSIAuthentication()
-        subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None)
-        if not subscription_id:
-            try:
-                # use the first subscription of the MSI
-                subscription_client = SubscriptionClient(credentials)
-                subscription = next(subscription_client.subscriptions.list())
-                subscription_id = str(subscription.subscription_id)
-            except Exception as exc:
-                self.fail("Failed to get MSI token: {0}. "
-                          "Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc)))
-        return {
-            'credentials': credentials,
-            'subscription_id': subscription_id
-        }
-
-    def _get_azure_cli_credentials(self):
-        credentials, subscription_id = get_azure_cli_credentials()
-        cloud_environment = get_cli_active_cloud()
-
-        cli_credentials = {
-            'credentials': credentials,
-            'subscription_id': subscription_id,
-            'cloud_environment': cloud_environment
-        }
-        return cli_credentials
-
-    def _get_env_credentials(self):
-        env_credentials = dict()
-        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
-            env_credentials[attribute] = os.environ.get(env_variable, None)
-
-        if env_credentials['profile']:
-            credentials = self._get_profile(env_credentials['profile'])
-            return credentials
-
-        if env_credentials.get('subscription_id') is not None:
-            return env_credentials
-
-        return None
-
-    def _get_credentials(self, params):
-        # Get authentication credentials.
-        self.log('Getting credentials')
-
-        arg_credentials = dict()
-        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
-            arg_credentials[attribute] = params.get(attribute, None)
-
-        auth_source = params.get('auth_source', None)
-        if not auth_source:
-            auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto')
-
-        if auth_source == 'msi':
-            self.log('Retrieving credenitals from MSI')
-            return self._get_msi_credentials(arg_credentials['subscription_id'])
-
-        if auth_source == 'cli':
-            if not HAS_AZURE_CLI_CORE:
-                self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`")
-            try:
-                self.log('Retrieving credentials from Azure CLI profile')
-                cli_credentials = self._get_azure_cli_credentials()
-                return cli_credentials
-            except CLIError as err:
-                self.fail("Azure CLI profile cannot be loaded - {0}".format(err))
-
-        if auth_source == 'env':
-            self.log('Retrieving credentials from environment')
-            env_credentials = self._get_env_credentials()
-            return env_credentials
-
-        if auth_source == 'credential_file':
-            self.log("Retrieving credentials from credential file")
-            profile = params.get('profile', 'default')
-            default_credentials = self._get_profile(profile)
-            return default_credentials
-
-        # auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials
-        # try module params
-        if arg_credentials['profile'] is not None:
-            self.log('Retrieving credentials with profile parameter.')
-            credentials = self._get_profile(arg_credentials['profile'])
-            return credentials
-
-        if arg_credentials['subscription_id']:
-            self.log('Received credentials from parameters.')
-            return arg_credentials
-
-        # try environment
-        env_credentials = self._get_env_credentials()
-        if env_credentials:
-            self.log('Received credentials from env.')
-            return env_credentials
-
-        # try default profile from ~./azure/credentials
-        default_credentials = self._get_profile()
-        if default_credentials:
-            self.log('Retrieved default profile credentials from ~/.azure/credentials.')
-            return default_credentials
-
-        try:
-            if HAS_AZURE_CLI_CORE:
-                self.log('Retrieving credentials from AzureCLI profile')
-            cli_credentials = self._get_azure_cli_credentials()
-            return cli_credentials
-        except CLIError as ce:
-            self.log('Error getting AzureCLI profile credentials - {0}'.format(ce))
-
-        return None
-
     def parse_resource_to_dict(self, resource):
         '''
         Return a dict of the give resource, which contains name and resource group.
@@ -947,9 +710,9 @@ class AzureRMModuleBase(object):
 
         if not base_url:
             # most things are resource_manager, don't make everyone specify
-            base_url = self._cloud_environment.endpoints.resource_manager
+            base_url = self.azure_auth._cloud_environment.endpoints.resource_manager
 
-        client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url)
+        client_kwargs = dict(credentials=self.azure_auth.azure_credentials, subscription_id=self.azure_auth.subscription_id, base_url=base_url)
 
         api_profile_dict = {}
 
@@ -992,11 +755,24 @@ class AzureRMModuleBase(object):
         if VSCODEEXT_USER_AGENT_KEY in os.environ:
             client.config.add_user_agent(os.environ[VSCODEEXT_USER_AGENT_KEY])
 
-        if self._cert_validation_mode == 'ignore':
+        if self.azure_auth._cert_validation_mode == 'ignore':
             client.config.session_configuration_callback = self._validation_ignore_callback
 
         return client
 
+    # passthru methods to AzureAuth instance for backcompat
+    @property
+    def credentials(self):
+        return self.azure_auth.credentials
+
+    @property
+    def _cloud_environment(self):
+        return self.azure_auth._cloud_environment
+
+    @property
+    def subscription_id(self):
+        return self.azure_auth.subscription_id
+
     @property
     def storage_client(self):
         self.log('Getting storage client...')
@@ -1152,3 +928,281 @@ class AzureRMModuleBase(object):
             self._monitor_client = self.get_mgmt_svc_client(MonitorManagementClient,
                                                             base_url=self._cloud_environment.endpoints.resource_manager)
         return self._monitor_client
+
+
+class AzureRMAuthException(Exception):
+    pass
+
+
+class AzureRMAuth(object):
+    def __init__(self, auth_source='auto', profile=None, subscription_id=None, client_id=None, secret=None,
+                 tenant=None, ad_user=None, password=None, cloud_environment='AzureCloud', cert_validation_mode='validate',
+                 api_profile='latest', adfs_authority_url=None, fail_impl=None, **kwargs):
+
+        if fail_impl:
+            self._fail_impl = fail_impl
+        else:
+            self._fail_impl = self._default_fail_impl
+
+        self._cloud_environment = None
+        self._adfs_authority_url = None
+
+        # authenticate
+        self.credentials = self._get_credentials(
+            dict(auth_source=auth_source, profile=profile, subscription_id=subscription_id, client_id=client_id, secret=secret,
+                 tenant=tenant, ad_user=ad_user, password=password, cloud_environment=cloud_environment,
+                 cert_validation_mode=cert_validation_mode, api_profile=api_profile, adfs_authority_url=adfs_authority_url))
+
+        if not self.credentials:
+            if HAS_AZURE_CLI_CORE:
+                self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
+                          "define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).")
+            else:
+                self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
+                          "define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).")
+
+        # cert validation mode precedence: module-arg, credential profile, env, "validate"
+        self._cert_validation_mode = cert_validation_mode or self.credentials.get('cert_validation_mode') or \
+            os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate'
+
+        if self._cert_validation_mode not in ['validate', 'ignore']:
+            self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode))
+
+        # if cloud_environment specified, look up/build Cloud object
+        raw_cloud_env = self.credentials.get('cloud_environment')
+        if self.credentials.get('credentials') is not None and raw_cloud_env is not None:
+            self._cloud_environment = raw_cloud_env
+        elif not raw_cloud_env:
+            self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD  # SDK default
+        else:
+            # try to look up "well-known" values via the name attribute on azure_cloud members
+            all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
+            matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
+            if len(matched_clouds) == 1:
+                self._cloud_environment = matched_clouds[0]
+            elif len(matched_clouds) > 1:
+                self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
+            else:
+                if not urlparse.urlparse(raw_cloud_env).scheme:
+                    self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
+                try:
+                    self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
+                except Exception as e:
+                    self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
+
+        if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None:
+            self.fail("Credentials did not include a subscription_id value.")
+        self.log("setting subscription_id")
+        self.subscription_id = self.credentials['subscription_id']
+
+        # get authentication authority
+        # for adfs, user could pass in authority or not.
+        # for others, use default authority from cloud environment
+        if self.credentials.get('adfs_authority_url') is None:
+            self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
+        else:
+            self._adfs_authority_url = self.credentials.get('adfs_authority_url')
+
+        # get resource from cloud environment
+        self._resource = self._cloud_environment.endpoints.active_directory_resource_id
+
+        if self.credentials.get('credentials') is not None:
+            # AzureCLI credentials
+            self.azure_credentials = self.credentials['credentials']
+        elif self.credentials.get('client_id') is not None and \
+                self.credentials.get('secret') is not None and \
+                self.credentials.get('tenant') is not None:
+                self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
+                                                                     secret=self.credentials['secret'],
+                                                                     tenant=self.credentials['tenant'],
+                                                                     cloud_environment=self._cloud_environment,
+                                                                     verify=self._cert_validation_mode == 'validate')
+
+        elif self.credentials.get('ad_user') is not None and \
+                self.credentials.get('password') is not None and \
+                self.credentials.get('client_id') is not None and \
+                self.credentials.get('tenant') is not None:
+
+                self.azure_credentials = self.acquire_token_with_username_password(
+                    self._adfs_authority_url,
+                    self._resource,
+                    self.credentials['ad_user'],
+                    self.credentials['password'],
+                    self.credentials['client_id'],
+                    self.credentials['tenant'])
+
+        elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
+            tenant = self.credentials.get('tenant')
+            if not tenant:
+                tenant = 'common'  # SDK default
+
+            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
+                                                         self.credentials['password'],
+                                                         tenant=tenant,
+                                                         cloud_environment=self._cloud_environment,
+                                                         verify=self._cert_validation_mode == 'validate')
+        else:
+            self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
+                      "Credentials must include client_id, secret and tenant or ad_user and password, or "
+                      "ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or "
+                      "be logged in using AzureCLI.")
+
+    def fail(self, msg, exception=None, **kwargs):
+        self._fail_impl(msg)
+
+    def _default_fail_impl(self, msg, exception=None, **kwargs):
+        raise AzureRMAuthException(msg)
+
+    def _get_profile(self, profile="default"):
+        path = expanduser("~/.azure/credentials")
+        try:
+            config = configparser.ConfigParser()
+            config.read(path)
+        except Exception as exc:
+            self.fail("Failed to access {0}. Check that the file exists and you have read "
+                      "access. {1}".format(path, str(exc)))
+        credentials = dict()
+        for key in AZURE_CREDENTIAL_ENV_MAPPING:
+            try:
+                credentials[key] = config.get(profile, key, raw=True)
+            except:
+                pass
+
+        if credentials.get('subscription_id'):
+            return credentials
+
+        return None
+
+    def _get_msi_credentials(self, subscription_id_param=None):
+        credentials = MSIAuthentication()
+        subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None)
+        if not subscription_id:
+            try:
+                # use the first subscription of the MSI
+                subscription_client = SubscriptionClient(credentials)
+                subscription = next(subscription_client.subscriptions.list())
+                subscription_id = str(subscription.subscription_id)
+            except Exception as exc:
+                self.fail("Failed to get MSI token: {0}. "
+                          "Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc)))
+        return {
+            'credentials': credentials,
+            'subscription_id': subscription_id
+        }
+
+    def _get_azure_cli_credentials(self):
+        credentials, subscription_id = get_azure_cli_credentials()
+        cloud_environment = get_cli_active_cloud()
+
+        cli_credentials = {
+            'credentials': credentials,
+            'subscription_id': subscription_id,
+            'cloud_environment': cloud_environment
+        }
+        return cli_credentials
+
+    def _get_env_credentials(self):
+        env_credentials = dict()
+        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
+            env_credentials[attribute] = os.environ.get(env_variable, None)
+
+        if env_credentials['profile']:
+            credentials = self._get_profile(env_credentials['profile'])
+            return credentials
+
+        if env_credentials.get('subscription_id') is not None:
+            return env_credentials
+
+        return None
+
+    # TODO: use explicit kwargs instead of intermediate dict
+    def _get_credentials(self, params):
+        # Get authentication credentials.
+        self.log('Getting credentials')
+
+        arg_credentials = dict()
+        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
+            arg_credentials[attribute] = params.get(attribute, None)
+
+        auth_source = params.get('auth_source', None)
+        if not auth_source:
+            auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto')
+
+        if auth_source == 'msi':
+            self.log('Retrieving credenitals from MSI')
+            return self._get_msi_credentials(arg_credentials['subscription_id'])
+
+        if auth_source == 'cli':
+            if not HAS_AZURE_CLI_CORE:
+                self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`")
+            try:
+                self.log('Retrieving credentials from Azure CLI profile')
+                cli_credentials = self._get_azure_cli_credentials()
+                return cli_credentials
+            except CLIError as err:
+                self.fail("Azure CLI profile cannot be loaded - {0}".format(err))
+
+        if auth_source == 'env':
+            self.log('Retrieving credentials from environment')
+            env_credentials = self._get_env_credentials()
+            return env_credentials
+
+        if auth_source == 'credential_file':
+            self.log("Retrieving credentials from credential file")
+            profile = params.get('profile', 'default')
+            default_credentials = self._get_profile(profile)
+            return default_credentials
+
+        # auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials
+        # try module params
+        if arg_credentials['profile'] is not None:
+            self.log('Retrieving credentials with profile parameter.')
+            credentials = self._get_profile(arg_credentials['profile'])
+            return credentials
+
+        if arg_credentials['subscription_id']:
+            self.log('Received credentials from parameters.')
+            return arg_credentials
+
+        # try environment
+        env_credentials = self._get_env_credentials()
+        if env_credentials:
+            self.log('Received credentials from env.')
+            return env_credentials
+
+        # try default profile from ~./azure/credentials
+        default_credentials = self._get_profile()
+        if default_credentials:
+            self.log('Retrieved default profile credentials from ~/.azure/credentials.')
+            return default_credentials
+
+        try:
+            if HAS_AZURE_CLI_CORE:
+                self.log('Retrieving credentials from AzureCLI profile')
+            cli_credentials = self._get_azure_cli_credentials()
+            return cli_credentials
+        except CLIError as ce:
+            self.log('Error getting AzureCLI profile credentials - {0}'.format(ce))
+
+        return None
+
+    def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant):
+        authority_uri = authority
+
+        if tenant is not None:
+            authority_uri = authority + '/' + tenant
+
+        context = AuthenticationContext(authority_uri)
+        token_response = context.acquire_token_with_username_password(resource, username, password, client_id)
+
+        return AADTokenCredentials(token_response)
+
+    def log(self, msg, pretty_print=False):
+        pass
+        # Use only during module development
+        # if self.debug:
+        #     log_file = open('azure_rm.log', 'a')
+        #     if pretty_print:
+        #         log_file.write(json.dumps(msg, indent=4, sort_keys=True))
+        #     else:
+        #         log_file.write(msg + u'\n')

lib/ansible/plugins/inventory/__init__.py

@@ -286,7 +286,7 @@ class Constructable(object):
                     composite = self._compose(compose[varname], variables)
                 except Exception as e:
                     if strict:
-                        raise AnsibleError("Could not set %s: %s" % (varname, to_native(e)))
+                        raise AnsibleError("Could not set %s for host %s: %s" % (varname, host, to_native(e)))
                     continue
                 self.inventory.set_variable(host, varname, composite)
 

lib/ansible/plugins/inventory/azure_rm.py

@@ -0,0 +1,540 @@
+# Copyright (c) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+    name: azure_rm
+    plugin_type: inventory
+    short_description: Azure Resource Manager inventory plugin
+    extends_documentation_fragment:
+      - azure
+    description:
+        - Query VM details from Azure Resource Manager
+        - Requires a YAML configuration file whose name ends with '.azure_rm.yaml'
+        - By default, sets C(ansible_host) to the first public IP address found (preferring the primary NIC). If no
+          public IPs are found, the first private IP (also preferring the primary NIC). The default may be overridden
+          via C(hostvar_expressions); see examples.
+    options:
+        plugin:
+            description: marks this as an instance of the 'azure_rm' plugin
+            required: true
+            choices: ['azure_rm']
+        include_vm_resource_groups:
+            description: A list of resource group names to search for virtual machines. '\*' will include all resource
+                groups in the subscription.
+            default: ['*']
+        include_vmss_resource_groups:
+            description: A list of resource group names to search for virtual machine scale sets (VMSSs). '\*' will
+                include all resource groups in the subscription.
+            default: []
+        fail_on_template_errors:
+            description: When false, template failures during group and filter processing are silently ignored (eg,
+                if a filter or group expression refers to an undefined host variable)
+            choices: [True, False]
+            default: True
+        keyed_groups:
+            description: Creates groups based on the value of a host variable. Requires a list of dictionaries,
+                defining C(key) (the source dictionary-typed variable), C(prefix) (the prefix to use for the new group
+                name), and optionally C(separator) (which defaults to C(_))
+        conditional_groups:
+            description: A mapping of group names to Jinja2 expressions. When the mapped expression is true, the host
+                is added to the named group.
+        hostvar_expressions:
+            description: A mapping of hostvar names to Jinja2 expressions. The value for each host is the result of the
+                Jinja2 expression (which may refer to any of the host's existing variables at the time this inventory
+                plugin runs).
+        exclude_host_filters:
+            description: Excludes hosts from the inventory with a list of Jinja2 conditional expressions. Each
+                expression in the list is evaluated for each host; when the expression is true, the host is excluded
+                from the inventory.
+            default: []
+        batch_fetch:
+            description: To improve performance, results are fetched using an unsupported batch API. Disabling
+                C(batch_fetch) uses a much slower serial fetch, resulting in many more round-trips. Generally only
+                useful for troubleshooting.
+            default: true
+        default_host_filters:
+            description: A default set of filters that is applied in addition to the conditions in
+                C(exclude_host_filters) to exclude powered-off and not-fully-provisioned hosts. Set this to a different
+                value or empty list if you need to include hosts in these states.
+            default: ['powerstate != "running"', 'provisioning_state != "succeeded"']
+'''
+
+EXAMPLES = '''
+# The following host variables are always available:
+# public_ipv4_addresses: all public IP addresses, with the primary IP config from the primary NIC first
+# public_dns_hostnames: all public DNS hostnames, with the primary IP config from the primary NIC first
+# private_ipv4_addresses: all private IP addressses, with the primary IP config from the primary NIC first
+# id: the VM's Azure resource ID, eg /subscriptions/00000000-0000-0000-1111-1111aaaabb/resourceGroups/my_rg/providers/Microsoft.Compute/virtualMachines/my_vm
+# location: the VM's Azure location, eg 'westus', 'eastus'
+# name: the VM's resource name, eg 'myvm'
+# powerstate: the VM's current power state, eg: 'running', 'stopped', 'deallocated'
+# provisioning_state: the VM's current provisioning state, eg: 'succeeded'
+# tags: dictionary of the VM's defined tag values
+# resource_type: the VM's resource type, eg: 'Microsoft.Compute/virtualMachine', 'Microsoft.Compute/virtualMachineScaleSets/virtualMachines'
+# vmid: the VM's internal SMBIOS ID, eg: '36bca69d-c365-4584-8c06-a62f4a1dc5d2'
+# vmss: if the VM is a member of a scaleset (vmss), a dictionary including the id and name of the parent scaleset
+
+
+# sample 'myazuresub.azure_rm.yaml'
+
+# required for all azure_rm inventory plugin configs
+plugin: azure_rm
+
+# forces this plugin to use a CLI auth session instead of the automatic auth source selection (eg, prevents the
+# presence of 'ANSIBLE_AZURE_RM_X' environment variables from overriding CLI auth)
+auth_source: cli
+
+# fetches VMs from an explicit list of resource groups instead of default all (- '*')
+include_vm_resource_groups:
+- myrg1
+- myrg2
+
+# fetches VMs from VMSSs in all resource groups (defaults to no VMSS fetch)
+include_vmss_resource_groups:
+- '*'
+
+# places a host in the named group if the associated condition evaluates to true
+conditional_groups:
+  # since this will be true for every host, every host sourced from this inventory plugin config will be in the
+  # group 'all_the_hosts'
+  all_the_hosts: true
+  # if the VM's "name" variable contains "dbserver", it will be placed in the 'db_hosts' group
+  db_hosts: "'dbserver' in name"
+
+# adds variables to each host found by this inventory plugin, whose values are the result of the associated expression
+hostvar_expressions:
+  my_host_var:
+  # A statically-valued expression has to be both single and double-quoted, or use escaped quotes, since the outer
+  # layer of quotes will be consumed by YAML. Without the second set of quotes, it interprets 'staticvalue' as a
+  # variable instead of a string literal.
+  some_statically_valued_var: "'staticvalue'"
+  # overrides the default ansible_host value with a custom Jinja2 expression, in this case, the first DNS hostname, or
+  # if none are found, the first public IP address.
+  ansible_host: (public_dns_hostnames + public_ipv4_addresses) | first
+
+# places hosts in dynamically-created groups based on a variable value.
+keyed_groups:
+# places each host in a group named 'tag_(tag name)_(tag value)' for each tag on a VM.
+- prefix: tag
+  key: tags
+# places each host in a group named 'azure_loc_(location name)', depending on the VM's location
+- prefix: azure_loc
+  key: location
+# places host in a group named 'some_tag_X' using the value of the 'sometag' tag on a VM as X, and defaulting to the
+# value 'none' (eg, the group 'some_tag_none') if the 'sometag' tag is not defined for a VM.
+- prefix: some_tag
+  key: tags.sometag | default('none')
+
+# excludes a host from the inventory when any of these expressions is true, can refer to any vars defined on the host
+exclude_host_filters:
+# excludes hosts in the eastus region
+- location in ['eastus']
+# excludes hosts that are powered off
+- powerstate != 'running'
+'''
+
+# FUTURE: do we need a set of sane default filters, separate from the user-defineable ones?
+# eg, powerstate==running, provisioning_state==succeeded
+
+
+import hashlib
+import json
+import re
+
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
+
+from collections import namedtuple
+from ansible import release
+from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
+from ansible.module_utils.six import iteritems
+from ansible.module_utils.azure_rm_common import AzureRMAuth
+from ansible.errors import AnsibleParserError, AnsibleError
+from ansible.module_utils.parsing.convert_bool import boolean
+from ansible.module_utils._text import to_native
+from itertools import chain
+from msrest import ServiceClient, Serializer, Deserializer
+from msrestazure import AzureConfiguration
+from msrestazure.polling.arm_polling import ARMPolling
+
+
+class AzureRMRestConfiguration(AzureConfiguration):
+    def __init__(self, credentials, subscription_id, base_url=None):
+
+        if credentials is None:
+            raise ValueError("Parameter 'credentials' must not be None.")
+        if subscription_id is None:
+            raise ValueError("Parameter 'subscription_id' must not be None.")
+        if not base_url:
+            base_url = 'https://management.azure.com'
+
+        super(AzureRMRestConfiguration, self).__init__(base_url)
+
+        self.add_user_agent('ansible-dynamic-inventory/{0}'.format(release.__version__))
+
+        self.credentials = credentials
+        self.subscription_id = subscription_id
+
+
+UrlAction = namedtuple('UrlAction', ['url', 'api_version', 'handler', 'handler_args'])
+
+
+# FUTURE: add Cacheable support once we have a sane serialization format
+class InventoryModule(BaseInventoryPlugin, Constructable):
+
+    NAME = 'azure_rm'
+
+    def __init__(self):
+        super(InventoryModule, self).__init__()
+
+        self._serializer = Serializer()
+        self._deserializer = Deserializer()
+        self._hosts = []
+        self._filters = None
+
+        # FUTURE: use API profiles with defaults
+        self._compute_api_version = '2017-03-30'
+        self._network_api_version = '2015-06-15'
+
+        self._default_header_parameters = {'Content-Type': 'application/json; charset=utf-8'}
+
+        self._request_queue = Queue()
+
+        self.azure_auth = None
+
+        self._batch_fetch = False
+
+    def verify_file(self, path):
+        '''
+            :param loader: an ansible.parsing.dataloader.DataLoader object
+            :param path: the path to the inventory config file
+            :return the contents of the config file
+        '''
+        if super(InventoryModule, self).verify_file(path):
+            if re.match(r'.+\.azure_rm\.y(a)?ml$', path):
+                return True
+        # display.debug("azure_rm inventory filename must match '*.azure_rm.yml' or '*.azure_rm.yaml'")
+        return False
+
+    def parse(self, inventory, loader, path, cache=True):
+        super(InventoryModule, self).parse(inventory, loader, path)
+
+        self._read_config_data(path)
+        self._batch_fetch = self.get_option('batch_fetch')
+
+        self._filters = self.get_option('exclude_host_filters') + self.get_option('default_host_filters')
+
+        try:
+            self._credential_setup()
+            self._get_hosts()
+        except Exception as ex:
+            raise
+
+    def _credential_setup(self):
+        auth_options = dict(
+            auth_source=self.get_option('auth_source'),
+            profile=self.get_option('profile'),
+            subscription_id=self.get_option('subscription_id'),
+            client_id=self.get_option('client_id'),
+            secret=self.get_option('secret'),
+            tenant=self.get_option('tenant'),
+            ad_user=self.get_option('ad_user'),
+            password=self.get_option('password'),
+            cloud_environment=self.get_option('cloud_environment'),
+            cert_validation_mode=self.get_option('cert_validation_mode'),
+            api_profile=self.get_option('api_profile'),
+            adfs_authority_url=self.get_option('adfs_authority_url')
+        )
+
+        self.azure_auth = AzureRMAuth(**auth_options)
+
+        self._clientconfig = AzureRMRestConfiguration(self.azure_auth.azure_credentials, self.azure_auth.subscription_id,
+                                                      self.azure_auth._cloud_environment.endpoints.resource_manager)
+        self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig)
+
+    def _enqueue_get(self, url, api_version, handler, handler_args=None):
+        if not handler_args:
+            handler_args = {}
+        self._request_queue.put_nowait(UrlAction(url=url, api_version=api_version, handler=handler, handler_args=handler_args))
+
+    def _enqueue_vm_list(self, rg='*'):
+        if not rg or rg == '*':
+            url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines'
+        else:
+            url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines'
+
+        url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg)
+        self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response)
+
+    def _enqueue_vmss_list(self, rg=None):
+        if not rg or rg == '*':
+            url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets'
+        else:
+            url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets'
+
+        url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg)
+        self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vmss_page_response)
+
+    def _get_hosts(self):
+        for vm_rg in self.get_option('include_vm_resource_groups'):
+            self._enqueue_vm_list(vm_rg)
+
+        for vmss_rg in self.get_option('include_vmss_resource_groups'):
+            self._enqueue_vmss_list(vmss_rg)
+
+        if self._batch_fetch:
+            self._process_queue_batch()
+        else:
+            self._process_queue_serial()
+
+        constructable_config_strict = boolean(self.get_option('fail_on_template_errors'))
+        constructable_config_compose = self.get_option('hostvar_expressions')
+        constructable_config_groups = self.get_option('conditional_groups')
+        constructable_config_keyed_groups = self.get_option('keyed_groups')
+
+        for h in self._hosts:
+            inventory_hostname = self._get_hostname(h)
+            if self._filter_host(inventory_hostname, h.hostvars):
+                continue
+            self.inventory.add_host(inventory_hostname)
+            # FUTURE: configurable default IP list? can already do this via hostvar_expressions
+            self.inventory.set_variable(inventory_hostname, "ansible_host",
+                                        next(chain(h.hostvars['public_ipv4_addresses'], h.hostvars['private_ipv4_addresses']), None))
+            for k, v in iteritems(h.hostvars):
+                # FUTURE: configurable hostvar prefix? Makes docs harder...
+                self.inventory.set_variable(inventory_hostname, k, v)
+
+            # constructable delegation
+            self._set_composite_vars(constructable_config_compose, h.hostvars, inventory_hostname, strict=constructable_config_strict)
+            self._add_host_to_composed_groups(constructable_config_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict)
+            self._add_host_to_keyed_groups(constructable_config_keyed_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict)
+
+    # FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host
+    def _filter_host(self, inventory_hostname, hostvars):
+        self.templar.set_available_variables(hostvars)
+
+        for condition in self._filters:
+            # FUTURE: should warn/fail if conditional doesn't return True or False
+            conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format(condition)
+            try:
+                if boolean(self.templar.template(conditional)):
+                    return True
+            except Exception as e:
+                if boolean(self.get_option('fail_on_template_errors')):
+                    raise AnsibleParserError("Error evaluating filter condition '{0}' for host {1}: {2}".format(condition, inventory_hostname, to_native(e)))
+                continue
+
+        return False
+
+    def _get_hostname(self, host):
+        # FUTURE: configurable hostname sources
+        return host.default_inventory_hostname
+
+    def _process_queue_serial(self):
+        try:
+            while True:
+                item = self._request_queue.get_nowait()
+                resp = self.send_request(item.url, item.api_version)
+                item.handler(resp, **item.handler_args)
+        except Empty:
+            pass
+
+    def _on_vm_page_response(self, response, vmss=None):
+        next_link = response.get('nextLink')
+
+        if next_link:
+            self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vm_page_response)
+
+        for h in response['value']:
+            # FUTURE: add direct VM filtering by tag here (performance optimization)?
+            self._hosts.append(AzureHost(h, self, vmss=vmss))
+
+    def _on_vmss_page_response(self, response):
+        next_link = response.get('nextLink')
+
+        if next_link:
+            self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vmss_page_response)
+
+        # FUTURE: add direct VMSS filtering by tag here (performance optimization)?
+        for vmss in response['value']:
+            url = '{0}/virtualMachines'.format(vmss['id'])
+            # VMSS instances look close enough to regular VMs that we can share the handler impl...
+            self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response, handler_args=dict(vmss=vmss))
+
+    # use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip
+    #
+    def _process_queue_batch(self):
+        while True:
+            batch_requests = []
+            batch_item_index = 0
+            batch_response_handlers = []
+            try:
+                while batch_item_index < 500:
+                    item = self._request_queue.get_nowait()
+
+                    query_parameters = {'api-version': item.api_version}
+                    req = self._client.get(item.url, query_parameters)
+
+                    batch_requests.append(dict(httpMethod="GET", url=req.url))
+                    batch_response_handlers.append(item)
+                    batch_item_index += 1
+            except Empty:
+                pass
+
+            if not batch_requests:
+                break
+
+            batch_resp = self._send_batch(batch_requests)
+
+            for idx, r in enumerate(batch_resp['responses']):
+                status_code = r.get('httpStatusCode')
+                if status_code != 200:
+                    # FUTURE: error-tolerant operation mode (eg, permissions)
+                    raise AnsibleError("a batched request failed with status code {0}, url {1}".format(status_code, batch_requests[idx].get('url')))
+
+                item = batch_response_handlers[idx]
+                # FUTURE: store/handle errors from individual handlers
+                item.handler(r['content'], **item.handler_args)
+
+    def _send_batch(self, batched_requests):
+        url = '/batch'
+        query_parameters = {'api-version': '2015-11-01'}
+
+        body_obj = dict(requests=batched_requests)
+
+        body_content = self._serializer.body(body_obj, 'object')
+
+        request = self._client.post(url, query_parameters)
+        initial_response = self._client.send(request, self._default_header_parameters, body_content)
+
+        # FUTURE: configurable timeout?
+        poller = ARMPolling(timeout=2)
+        poller.initialize(client=self._client,
+                          initial_response=initial_response,
+                          deserialization_callback=lambda r: self._deserializer('object', r))
+
+        poller.run()
+
+        return poller.resource()
+
+    def send_request(self, url, api_version):
+        query_parameters = {'api-version': api_version}
+        req = self._client.get(url, query_parameters)
+        resp = self._client.send(req, self._default_header_parameters, stream=False)
+
+        resp.raise_for_status()
+        content = resp.content
+
+        return json.loads(content)
+
+
+# VM list (all, N resource groups): VM -> InstanceView, N NICs, N PublicIPAddress)
+# VMSS VMs (all SS, N specific SS, N resource groups?): SS -> VM -> InstanceView, N NICs, N PublicIPAddress)
+
+class AzureHost(object):
+    _powerstate_regex = re.compile('^PowerState/(?P<powerstate>.+)$')
+
+    def __init__(self, vm_model, inventory_client, vmss=None):
+        self._inventory_client = inventory_client
+        self._vm_model = vm_model
+        self._vmss = vmss
+
+        self._instanceview = None
+
+        self._powerstate = "unknown"
+        self.nics = []
+
+        # Azure often doesn't provide a globally-unique filename, so use resource name + a chunk of ID hash
+        self.default_inventory_hostname = '{0}_{1}'.format(vm_model['name'], hashlib.sha1(vm_model['id']).hexdigest()[0:4])
+
+        self._hostvars = {}
+
+        inventory_client._enqueue_get(url="{0}/instanceView".format(vm_model['id']),
+                                      api_version=self._inventory_client._compute_api_version,
+                                      handler=self._on_instanceview_response)
+
+        nic_refs = vm_model['properties']['networkProfile']['networkInterfaces']
+        for nic in nic_refs:
+            # single-nic instances don't set primary, so figure it out...
+            is_primary = nic.get('properties', {}).get('primary', len(nic_refs) == 1)
+            inventory_client._enqueue_get(url=nic['id'], api_version=self._inventory_client._network_api_version,
+                                          handler=self._on_nic_response,
+                                          handler_args=dict(is_primary=is_primary))
+
+    @property
+    def hostvars(self):
+        if self._hostvars != {}:
+            return self._hostvars
+
+        new_hostvars = dict(
+            public_ipv4_addresses=[],
+            public_dns_hostnames=[],
+            private_ipv4_addresses=[],
+            id=self._vm_model['id'],
+            location=self._vm_model['location'],
+            name=self._vm_model['name'],
+            powerstate=self._powerstate,
+            provisioning_state=self._vm_model['properties']['provisioningState'].lower(),
+            tags=self._vm_model.get('tags', {}),
+            resource_type=self._vm_model.get('type', "unknown"),
+            vmid=self._vm_model['properties']['vmId'],
+            vmss=dict(
+                id=self._vmss['id'],
+                name=self._vmss['name'],
+            ) if self._vmss else {}
+        )
+
+        # set nic-related values from the primary NIC first
+        for nic in sorted(self.nics, key=lambda n: n.is_primary, reverse=True):
+            # and from the primary IP config per NIC first
+            for ipc in sorted(nic._nic_model['properties']['ipConfigurations'], key=lambda i: i['properties']['primary'], reverse=True):
+                private_ip = ipc['properties'].get('privateIPAddress')
+                if private_ip:
+                    new_hostvars['private_ipv4_addresses'].append(private_ip)
+                pip_id = ipc['properties'].get('publicIPAddress', {}).get('id')
+                if pip_id:
+                    pip = nic.public_ips[pip_id]
+                    new_hostvars['public_ipv4_addresses'].append(pip._pip_model['properties']['ipAddress'])
+                    pip_fqdn = pip._pip_model['properties'].get('dnsSettings', {}).get('fqdn')
+                    if pip_fqdn:
+                        new_hostvars['public_dns_hostnames'].append(pip_fqdn)
+
+        self._hostvars = new_hostvars
+
+        return self._hostvars
+
+    def _on_instanceview_response(self, vm_instanceview_model):
+        self._instanceview = vm_instanceview_model
+        self._powerstate = next((self._powerstate_regex.match(s.get('code', '')).group('powerstate')
+                                 for s in vm_instanceview_model.get('statuses', []) if self._powerstate_regex.match(s.get('code', ''))), 'unknown')
+
+    def _on_nic_response(self, nic_model, is_primary=False):
+        nic = AzureNic(nic_model=nic_model, inventory_client=self._inventory_client, is_primary=is_primary)
+        self.nics.append(nic)
+
+
+class AzureNic(object):
+    def __init__(self, nic_model, inventory_client, is_primary=False):
+        self._nic_model = nic_model
+        self.is_primary = is_primary
+        self._inventory_client = inventory_client
+
+        self.public_ips = {}
+
+        for ipc in nic_model['properties']['ipConfigurations']:
+            pip = ipc['properties'].get('publicIPAddress')
+            if pip:
+                self._inventory_client._enqueue_get(url=pip['id'], api_version=self._inventory_client._network_api_version, handler=self._on_pip_response)
+
+    def _on_pip_response(self, pip_model):
+        self.public_ips[pip_model['id']] = AzurePip(pip_model)
+
+
+class AzurePip(object):
+    def __init__(self, pip_model):
+        self._pip_model = pip_model