From b3f2d1befe509614febc28d1bf1734cb2cc50ac0 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 29 Aug 2017 10:35:24 -0700 Subject: [PATCH] expose cloud_environment override in azure_rm modules (#28743) * Can be set via env, credential profile, or module arg * Valid values defined by Azure Python SDK, currently `AzureCloud`,`AzureChinaCloud`,`AzureUSGovernment`,`AzureGermanCloud` or any Azure Stack metadata discovery URL. --- CHANGELOG.md | 2 + contrib/inventory/azure_rm.py | 74 ++++++++++++++----- docs/docsite/rst/guide_azure.rst | 8 ++ docs/docsite/rst/roadmap/ROADMAP_2_4.rst | 4 +- lib/ansible/module_utils/azure_rm_common.py | 64 +++++++++++----- .../cloud/azure/azure_rm_virtualmachine.py | 20 ++--- .../utils/module_docs_fragments/azure.py | 13 +++- 7 files changed, 131 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6da3c32471a..ba12cb6a12a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,8 @@ Ansible Changes By Release template hardcoded this to true. - Added a new parameter to command module that lets users specify data to pipe into the command's stdin. +- The azure_rm modules now accept a `cloud_environment` arg to access regional and private clouds. +- The azure_rm modules now require at least version 2.0.0 of the Azure Python SDK. ### New Modules diff --git a/contrib/inventory/azure_rm.py b/contrib/inventory/azure_rm.py index 2b9dbb10822..9b5ca1fbc92 100755 --- a/contrib/inventory/azure_rm.py +++ b/contrib/inventory/azure_rm.py @@ -49,6 +49,7 @@ Command line arguments: - tenant - ad_user - password + - cloud_environment Environment variables: - AZURE_PROFILE @@ -58,6 +59,7 @@ Environment variables: - AZURE_TENANT - AZURE_AD_USER - AZURE_PASSWORD + - AZURE_CLOUD_ENVIRONMENT Run for Specific Host ----------------------- @@ -190,22 +192,27 @@ import json import os import re import sys +import inspect +import traceback + from packaging.version import Version from os.path import expanduser +import ansible.module_utils.six.moves.urllib.parse as urlparse HAS_AZURE = True HAS_AZURE_EXC = None try: from msrestazure.azure_exceptions import CloudError + from msrestazure import azure_cloud from azure.mgmt.compute import __version__ as azure_compute_version from azure.common import AzureMissingResourceHttpError, AzureHttpError from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials - from azure.mgmt.network.network_management_client import NetworkManagementClient - from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient - from azure.mgmt.compute.compute_management_client import ComputeManagementClient + from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.resource.resources import ResourceManagementClient + from azure.mgmt.compute import ComputeManagementClient except ImportError as exc: HAS_AZURE_EXC = exc HAS_AZURE = False @@ -218,7 +225,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict( secret='AZURE_SECRET', tenant='AZURE_TENANT', ad_user='AZURE_AD_USER', - password='AZURE_PASSWORD' + password='AZURE_PASSWORD', + cloud_environment='AZURE_CLOUD_ENVIRONMENT', ) AZURE_CONFIG_SETTINGS = dict( @@ -232,7 +240,7 @@ AZURE_CONFIG_SETTINGS = dict( group_by_tag='AZURE_GROUP_BY_TAG' ) -AZURE_MIN_VERSION = "0.30.0rc5" +AZURE_MIN_VERSION = "2.0.0" def azure_id_to_dict(id): @@ -249,6 +257,7 @@ class AzureRM(object): def __init__(self, args): self._args = args + self._cloud_environment = None self._compute_client = None self._resource_client = None self._network_client = None @@ -262,6 +271,26 @@ class AzureRM(object): self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " "or define a profile in ~/.azure/credentials.") + # if cloud_environment specified, look up/build Cloud object + raw_cloud_env = self.credentials.get('cloud_environment') + if not raw_cloud_env: + self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default + else: + # try to look up "well-known" values via the name attribute on azure_cloud members + all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] + matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] + if len(matched_clouds) == 1: + self._cloud_environment = matched_clouds[0] + elif len(matched_clouds) > 1: + self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) + else: + if not urlparse.urlparse(raw_cloud_env).scheme: + self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) + try: + self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) + except Exception as e: + self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message)) + if self.credentials.get('subscription_id', None) is None: self.fail("Credentials did not include a subscription_id value.") self.log("setting subscription_id") @@ -272,13 +301,16 @@ class AzureRM(object): self.credentials.get('tenant') is not None: self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], secret=self.credentials['secret'], - tenant=self.credentials['tenant']) + tenant=self.credentials['tenant'], + cloud_environment=self._cloud_environment) elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: tenant = self.credentials.get('tenant') - if tenant is not None: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant) - else: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + if not tenant: + tenant = 'common' + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], + self.credentials['password'], + tenant=tenant, + cloud_environment=self._cloud_environment) else: self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " "Credentials must include client_id, secret and tenant or ad_user and password.") @@ -345,6 +377,10 @@ class AzureRM(object): self.log('Received credentials from parameters.') return arg_credentials + if arg_credentials['ad_user'] is not None: + self.log('Received credentials from parameters.') + return arg_credentials + # try environment env_credentials = self._get_env_credentials() if env_credentials: @@ -376,7 +412,7 @@ class AzureRM(object): def network_client(self): self.log('Getting network client') if not self._network_client: - self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id) + self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management) self._register('Microsoft.Network') return self._network_client @@ -384,14 +420,16 @@ class AzureRM(object): def rm_client(self): self.log('Getting resource manager client') if not self._resource_client: - self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id) + self._resource_client = ResourceManagementClient(self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.management) return self._resource_client @property def compute_client(self): self.log('Getting compute client') if not self._compute_client: - self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id) + self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management) self._register('Microsoft.Compute') return self._compute_client @@ -469,10 +507,12 @@ class AzureInventory(object): help='Azure Client Secret') parser.add_argument('--tenant', action='store', help='Azure Tenant Id') - parser.add_argument('--ad-user', action='store', + parser.add_argument('--ad_user', action='store', help='Active Directory User') parser.add_argument('--password', action='store', help='password') + parser.add_argument('--cloud_environment', action='store', + help='Azure Cloud Environment name or metadata discovery URL') parser.add_argument('--resource-groups', action='store', help='Return inventory for comma separated list of resource group names') parser.add_argument('--tags', action='store', @@ -793,11 +833,7 @@ class AzureInventory(object): def main(): if not HAS_AZURE: - sys.exit("The Azure python sdk is not installed (try `pip install 'azure>=2.0.0rc5' --upgrade`) - {0}".format(HAS_AZURE_EXC)) - - if Version(azure_compute_version) < Version(AZURE_MIN_VERSION): - sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} " - "Do you have Azure >= 2.0.0rc5 installed? (try `pip install 'azure>=2.0.0rc5' --upgrade`)".format(AZURE_MIN_VERSION, azure_compute_version)) + sys.exit("The Azure python sdk is not installed (try `pip install 'azure>={0}' --upgrade`) - {1}".format(AZURE_MIN_VERSION, HAS_AZURE_EXC)) AzureInventory() diff --git a/docs/docsite/rst/guide_azure.rst b/docs/docsite/rst/guide_azure.rst index db345079fef..f0b7831cb5e 100644 --- a/docs/docsite/rst/guide_azure.rst +++ b/docs/docsite/rst/guide_azure.rst @@ -119,6 +119,14 @@ Or, pass the following parameters for Active Directory username/password: * subscription_id +Other Cloud Environments +------------------------ + +To use an Azure Cloud other than the default public cloud (eg, Azure China Cloud, Azure US Government Cloud, Azure Stack), +pass the "cloud_environment" argument to modules, configure it in a credential profile, or set the "AZURE_CLOUD_ENVIRONMENT" +environment variable. The value is either a cloud name as defined by the Azure Python SDK (eg, "AzureChinaCloud", +"AzureUSGovernment"; defaults to "AzureCloud") or an Azure metadata discovery URL (for Azure Stack). + Creating Virtual Machines ------------------------- diff --git a/docs/docsite/rst/roadmap/ROADMAP_2_4.rst b/docs/docsite/rst/roadmap/ROADMAP_2_4.rst index 396b6f7e66e..afe29e680a8 100644 --- a/docs/docsite/rst/roadmap/ROADMAP_2_4.rst +++ b/docs/docsite/rst/roadmap/ROADMAP_2_4.rst @@ -174,8 +174,8 @@ AWS Azure ----- -- Expose endpoint overrides **(in progress)** -- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks) +- Expose endpoint overrides **(done)** +- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks) **(pushed to future)** - Add load balancer module **(in progress)** - Add Azure Functions module **(in progress)** diff --git a/lib/ansible/module_utils/azure_rm_common.py b/lib/ansible/module_utils/azure_rm_common.py index 7acf14d00f6..4e8d84b1406 100644 --- a/lib/ansible/module_utils/azure_rm_common.py +++ b/lib/ansible/module_utils/azure_rm_common.py @@ -23,12 +23,14 @@ import sys import copy import importlib import inspect +import traceback from packaging.version import Version from os.path import expanduser from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.six.moves import configparser +import ansible.module_utils.six.moves.urllib.parse as urlparse AZURE_COMMON_ARGS = dict( cli_default_profile=dict(type='bool'), @@ -39,6 +41,7 @@ AZURE_COMMON_ARGS = dict( tenant=dict(type='str', no_log=True), ad_user=dict(type='str', no_log=True), password=dict(type='str', no_log=True), + cloud_environment=dict(type='str'), # debug=dict(type='bool', default=False), ) @@ -50,7 +53,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict( secret='AZURE_SECRET', tenant='AZURE_TENANT', ad_user='AZURE_AD_USER', - password='AZURE_PASSWORD' + password='AZURE_PASSWORD', + cloud_environment='AZURE_CLOUD_ENVIRONMENT', ) AZURE_TAG_ARGS = dict( @@ -87,6 +91,7 @@ except ImportError as exc: try: from enum import Enum from msrestazure.azure_exceptions import CloudError + from msrestazure import azure_cloud from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, SecurityRule, NetworkInterface, \ NetworkInterfaceIPConfiguration, Subnet from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials @@ -173,6 +178,7 @@ class AzureRMModuleBase(object): self.fail("Do you have azure>={1} installed? Try `pip install 'azure>={1}' --upgrade`" "- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE)) + self._cloud_environment = None self._network_client = None self._storage_client = None self._resource_client = None @@ -188,6 +194,26 @@ class AzureRMModuleBase(object): self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " "or define a profile in ~/.azure/credentials or be logged using AzureCLI.") + # if cloud_environment specified, look up/build Cloud object + raw_cloud_env = self.credentials.get('cloud_environment') + if not raw_cloud_env: + self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default + else: + # try to look up "well-known" values via the name attribute on azure_cloud members + all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] + matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] + if len(matched_clouds) == 1: + self._cloud_environment = matched_clouds[0] + elif len(matched_clouds) > 1: + self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) + else: + if not urlparse.urlparse(raw_cloud_env).scheme: + self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) + try: + self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) + except Exception as e: + self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e)) + if self.credentials.get('subscription_id', None) is None: self.fail("Credentials did not include a subscription_id value.") self.log("setting subscription_id") @@ -198,24 +224,23 @@ class AzureRMModuleBase(object): self.credentials.get('tenant') is not None: self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], secret=self.credentials['secret'], - tenant=self.credentials['tenant']) + tenant=self.credentials['tenant'], + cloud_environment=self._cloud_environment) + elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: tenant = self.credentials.get('tenant') - if tenant is not None: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant) - else: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + if not tenant: + tenant = 'common' # SDK default + + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], + self.credentials['password'], + tenant=tenant, + cloud_environment=self._cloud_environment) else: self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " "Credentials must include client_id, secret and tenant or ad_user and password or " "be logged using AzureCLI.") - # base_url for sovereign cloud support. For now only if AzureCLI - if self.credentials.get('base_url') is not None: - self.base_url = self.credentials.get('base_url') - else: - self.base_url = None - # common parameter validation if self.module.params.get('tags'): self.validate_tags(self.module.params['tags']) @@ -340,11 +365,10 @@ class AzureRMModuleBase(object): self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`") try: credentials, subscription_id = get_azure_cli_credentials() - base_url = get_cli_active_cloud().endpoints.resource_manager + self._cloud_environment = get_cli_active_cloud() return { 'credentials': credentials, - 'subscription_id': subscription_id, - 'base_url': base_url + 'subscription_id': subscription_id } except CLIError as err: self.fail("AzureCLI profile cannot be loaded - {0}".format(err)) @@ -641,7 +665,7 @@ class AzureRMModuleBase(object): self._storage_client = StorageManagementClient( self.azure_credentials, self.subscription_id, - base_url=self.base_url, + base_url=self._cloud_environment.endpoints.resource_manager, api_version='2017-06-01' ) self._register('Microsoft.Storage') @@ -655,7 +679,7 @@ class AzureRMModuleBase(object): self._network_client = NetworkManagementClient( self.azure_credentials, self.subscription_id, - base_url=self.base_url, + base_url=self._cloud_environment.endpoints.resource_manager, api_version='2017-06-01' ) self._register('Microsoft.Network') @@ -669,7 +693,7 @@ class AzureRMModuleBase(object): self._resource_client = ResourceManagementClient( self.azure_credentials, self.subscription_id, - base_url=self.base_url, + base_url=self._cloud_environment.endpoints.resource_manager, api_version='2017-05-10' ) return self._resource_client @@ -682,7 +706,7 @@ class AzureRMModuleBase(object): self._compute_client = ComputeManagementClient( self.azure_credentials, self.subscription_id, - base_url=self.base_url, + base_url=self._cloud_environment.endpoints.resource_manager, api_version='2017-03-30' ) self._register('Microsoft.Compute') @@ -696,7 +720,7 @@ class AzureRMModuleBase(object): self._dns_client = DnsManagementClient( self.azure_credentials, self.subscription_id, - base_url=self.base_url + base_url=self._cloud_environment.endpoints.resource_manager, ) self._register('Microsoft.Dns') return self._dns_client diff --git a/lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py b/lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py index a2f6787a1d1..5bd2f9aad2a 100644 --- a/lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py +++ b/lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py @@ -430,10 +430,10 @@ AZURE_OBJECT_CLASS = 'VirtualMachine' AZURE_ENUM_MODULES = ['azure.mgmt.compute.models'] -def extract_names_from_blob_uri(blob_uri): +def extract_names_from_blob_uri(blob_uri, storage_suffix): # HACK: ditch this once python SDK supports get by URI - m = re.match('^https://(?P[^\.]+)\.blob\.core\.windows\.net/' - '(?P[^/]+)/(?P.+)$', blob_uri) + m = re.match('^https://(?P[^\.]+)\.blob\.{0}/' + '(?P[^/]+)/(?P.+)$'.format(storage_suffix), blob_uri) if not m: raise Exception("unable to parse blob uri '%s'" % blob_uri) extracted_names = m.groupdict() @@ -574,9 +574,10 @@ class AzureRMVirtualMachine(AzureRMModuleBase): if self.storage_account_name: self.get_storage_account(self.storage_account_name) - requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format(self.storage_account_name, - self.storage_container_name, - self.storage_blob_name) + requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format(self.storage_account_name, + self._cloud_environment.suffixes.storage_endpoint, + self.storage_container_name, + self.storage_blob_name) disable_ssh_password = not self.ssh_password_enabled @@ -689,8 +690,9 @@ class AzureRMVirtualMachine(AzureRMModuleBase): storage_account = self.create_default_storage_account() self.log("storage account:") self.log(self.serialize_obj(storage_account, 'StorageAccount'), pretty_print=True) - requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format( + requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format( storage_account.name, + self._cloud_environment.suffixes.storage_endpoint, self.storage_container_name, self.storage_blob_name) @@ -767,7 +769,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase): vm_dict['properties']['storageProfile']['osDisk']['name'], vhd, vm_dict['properties']['storageProfile']['osDisk']['createOption'], - os_type=vm_dict['properties']['storageProfile']['osDisk']['osType'], + vm_dict['properties']['storageProfile']['osDisk']['osType'], caching=vm_dict['properties']['storageProfile']['osDisk']['caching'] ), image_reference=ImageReference( @@ -1028,7 +1030,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase): for uri in vhd_uris: self.log("Extracting info from blob uri '{0}'".format(uri)) try: - blob_parts = extract_names_from_blob_uri(uri) + blob_parts = extract_names_from_blob_uri(uri, self._cloud_environment.suffixes.storage_endpoint) except Exception as exc: self.fail("Error parsing blob URI {0}".format(str(exc))) storage_account_name = blob_parts['accountname'] diff --git a/lib/ansible/utils/module_docs_fragments/azure.py b/lib/ansible/utils/module_docs_fragments/azure.py index e4c58a9698a..dcf25bba315 100644 --- a/lib/ansible/utils/module_docs_fragments/azure.py +++ b/lib/ansible/utils/module_docs_fragments/azure.py @@ -61,17 +61,22 @@ options: - Azure tenant ID. Use when authenticating with a Service Principal. required: false default: null - + cloud_environment: + description: + - For cloud environments other than the US public cloud, the environment name (as defined by Azure Python SDK, eg, C(AzureChinaCloud), + C(AzureUSGovernment)), or a metadata discovery endpoint URL (required for Azure Stack). Can also be set via credential file profile or + the C(AZURE_CLOUD_ENVIRONMENT) environment variable. + default: AzureCloud requirements: - "python >= 2.7" - - "azure == 2.0.0rc5" + - "azure >= 2.0.0" notes: - For authentication with Azure you can pass parameters, set environment variables or use a profile stored in ~/.azure/credentials. Authentication is possible using a service principal or Active Directory user. - To authenticate via service principal pass subscription_id, client_id, secret and tenant or set set environment + To authenticate via service principal, pass subscription_id, client_id, secret and tenant or set environment variables AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID, AZURE_SECRET and AZURE_TENANT. - - To Authentication via Active Directory user pass ad_user and password, or set AZURE_AD_USER and + - To authenticate via Active Directory user, pass ad_user and password, or set AZURE_AD_USER and AZURE_PASSWORD in the environment. - "Alternatively, credentials can be stored in ~/.azure/credentials. This is an ini file containing a [default] section and the following keys: subscription_id, client_id, secret and tenant or