cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

expose cloud_environment override in azure_rm modules (#28743)

Browse source 
* Can be set via env, credential profile, or module arg
* Valid values defined by Azure Python SDK, currently `AzureCloud`,`AzureChinaCloud`,`AzureUSGovernment`,`AzureGermanCloud` or any Azure Stack metadata discovery URL.
This commit is contained in:
Matt Davis 2017-08-29 10:35:24 -07:00 committed by ansibot
parent 6aaa0c3252
commit b3f2d1befe
7 changed files with 131 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -172,6 +172,8 @@ Ansible Changes By Release
template hardcoded this to true.
- Added a new parameter to command module that lets users specify data to pipe
into the command's stdin.
- The azure_rm modules now accept a `cloud_environment` arg to access regional and private clouds.
- The azure_rm modules now require at least version 2.0.0 of the Azure Python SDK.
### New Modules

74
contrib/inventory/azure_rm.py
View file

@ -49,6 +49,7 @@ Command line arguments:
- tenant
- ad_user
- password
- cloud_environment
Environment variables:
- AZURE_PROFILE
@ -58,6 +59,7 @@ Environment variables:
- AZURE_TENANT
- AZURE_AD_USER
- AZURE_PASSWORD
- AZURE_CLOUD_ENVIRONMENT
Run for Specific Host
-----------------------
@ -190,22 +192,27 @@ import json
import os
import re
import sys
import inspect
import traceback
from packaging.version import Version
from os.path import expanduser
import ansible.module_utils.six.moves.urllib.parse as urlparse
HAS_AZURE = True
HAS_AZURE_EXC = None
try:
from msrestazure.azure_exceptions import CloudError
from msrestazure import azure_cloud
from azure.mgmt.compute import __version__ as azure_compute_version
from azure.common import AzureMissingResourceHttpError, AzureHttpError
from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
from azure.mgmt.network.network_management_client import NetworkManagementClient
from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient
from azure.mgmt.compute.compute_management_client import ComputeManagementClient
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.resource.resources import ResourceManagementClient
from azure.mgmt.compute import ComputeManagementClient
except ImportError as exc:
HAS_AZURE_EXC = exc
HAS_AZURE = False
@ -218,7 +225,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
secret='AZURE_SECRET',
tenant='AZURE_TENANT',
ad_user='AZURE_AD_USER',
password='AZURE_PASSWORD'
password='AZURE_PASSWORD',
cloud_environment='AZURE_CLOUD_ENVIRONMENT',
)
AZURE_CONFIG_SETTINGS = dict(
@ -232,7 +240,7 @@ AZURE_CONFIG_SETTINGS = dict(
group_by_tag='AZURE_GROUP_BY_TAG'
)
AZURE_MIN_VERSION = "0.30.0rc5"
AZURE_MIN_VERSION = "2.0.0"
def azure_id_to_dict(id):
@ -249,6 +257,7 @@ class AzureRM(object):
def __init__(self, args):
self._args = args
self._cloud_environment = None
self._compute_client = None
self._resource_client = None
self._network_client = None
@ -262,6 +271,26 @@ class AzureRM(object):
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"or define a profile in ~/.azure/credentials.")
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message))
if self.credentials.get('subscription_id', None) is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
@ -272,13 +301,16 @@ class AzureRM(object):
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'])
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment)
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if tenant is not None:
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant)
else:
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
if not tenant:
tenant = 'common'
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment)
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password.")
@ -345,6 +377,10 @@ class AzureRM(object):
self.log('Received credentials from parameters.')
return arg_credentials
if arg_credentials['ad_user'] is not None:
self.log('Received credentials from parameters.')
return arg_credentials
# try environment
env_credentials = self._get_env_credentials()
if env_credentials:
@ -376,7 +412,7 @@ class AzureRM(object):
def network_client(self):
self.log('Getting network client')
if not self._network_client:
self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id)
self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management)
self._register('Microsoft.Network')
return self._network_client
@ -384,14 +420,16 @@ class AzureRM(object):
def rm_client(self):
self.log('Getting resource manager client')
if not self._resource_client:
self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id)
self._resource_client = ResourceManagementClient(self.azure_credentials,
self.subscription_id,
base_url=self._cloud_environment.endpoints.management)
return self._resource_client
@property
def compute_client(self):
self.log('Getting compute client')
if not self._compute_client:
self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id)
self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id, base_url=self._cloud_environment.endpoints.management)
self._register('Microsoft.Compute')
return self._compute_client
@ -469,10 +507,12 @@ class AzureInventory(object):
help='Azure Client Secret')
parser.add_argument('--tenant', action='store',
help='Azure Tenant Id')
parser.add_argument('--ad-user', action='store',
parser.add_argument('--ad_user', action='store',
help='Active Directory User')
parser.add_argument('--password', action='store',
help='password')
parser.add_argument('--cloud_environment', action='store',
help='Azure Cloud Environment name or metadata discovery URL')
parser.add_argument('--resource-groups', action='store',
help='Return inventory for comma separated list of resource group names')
parser.add_argument('--tags', action='store',
@ -793,11 +833,7 @@ class AzureInventory(object):
def main():
if not HAS_AZURE:
sys.exit("The Azure python sdk is not installed (try `pip install 'azure>=2.0.0rc5' --upgrade`) - {0}".format(HAS_AZURE_EXC))
if Version(azure_compute_version) < Version(AZURE_MIN_VERSION):
sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} "
"Do you have Azure >= 2.0.0rc5 installed? (try `pip install 'azure>=2.0.0rc5' --upgrade`)".format(AZURE_MIN_VERSION, azure_compute_version))
sys.exit("The Azure python sdk is not installed (try `pip install 'azure>={0}' --upgrade`) - {1}".format(AZURE_MIN_VERSION, HAS_AZURE_EXC))
AzureInventory()

8
docs/docsite/rst/guide_azure.rst
View file

@ -119,6 +119,14 @@ Or, pass the following parameters for Active Directory username/password:
* subscription_id
Other Cloud Environments
------------------------
To use an Azure Cloud other than the default public cloud (eg, Azure China Cloud, Azure US Government Cloud, Azure Stack),
pass the "cloud_environment" argument to modules, configure it in a credential profile, or set the "AZURE_CLOUD_ENVIRONMENT"
environment variable. The value is either a cloud name as defined by the Azure Python SDK (eg, "AzureChinaCloud",
"AzureUSGovernment"; defaults to "AzureCloud") or an Azure metadata discovery URL (for Azure Stack).
Creating Virtual Machines
-------------------------

4
docs/docsite/rst/roadmap/ROADMAP_2_4.rst
View file

@ -174,8 +174,8 @@ AWS
Azure
-----
- Expose endpoint overrides **(in progress)**
- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks)
- Expose endpoint overrides **(done)**
- Reformat/document module output to collapse internal API structures and surface important data (eg, public IPs, NICs, data disks) **(pushed to future)**
- Add load balancer module **(in progress)**
- Add Azure Functions module **(in progress)**

64
lib/ansible/module_utils/azure_rm_common.py
View file

@ -23,12 +23,14 @@ import sys
import copy
import importlib
import inspect
import traceback
from packaging.version import Version
from os.path import expanduser
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.six.moves import configparser
import ansible.module_utils.six.moves.urllib.parse as urlparse
AZURE_COMMON_ARGS = dict(
cli_default_profile=dict(type='bool'),
@ -39,6 +41,7 @@ AZURE_COMMON_ARGS = dict(
tenant=dict(type='str', no_log=True),
ad_user=dict(type='str', no_log=True),
password=dict(type='str', no_log=True),
cloud_environment=dict(type='str'),
# debug=dict(type='bool', default=False),
)
@ -50,7 +53,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
secret='AZURE_SECRET',
tenant='AZURE_TENANT',
ad_user='AZURE_AD_USER',
password='AZURE_PASSWORD'
password='AZURE_PASSWORD',
cloud_environment='AZURE_CLOUD_ENVIRONMENT',
)
AZURE_TAG_ARGS = dict(
@ -87,6 +91,7 @@ except ImportError as exc:
try:
from enum import Enum
from msrestazure.azure_exceptions import CloudError
from msrestazure import azure_cloud
from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, SecurityRule, NetworkInterface, \
NetworkInterfaceIPConfiguration, Subnet
from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
@ -173,6 +178,7 @@ class AzureRMModuleBase(object):
self.fail("Do you have azure>={1} installed? Try `pip install 'azure>={1}' --upgrade`"
"- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
self._cloud_environment = None
self._network_client = None
self._storage_client = None
self._resource_client = None
@ -188,6 +194,26 @@ class AzureRMModuleBase(object):
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"or define a profile in ~/.azure/credentials or be logged using AzureCLI.")
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
if self.credentials.get('subscription_id', None) is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
@ -198,24 +224,23 @@ class AzureRMModuleBase(object):
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'])
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment)
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if tenant is not None:
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'], tenant=tenant)
else:
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
if not tenant:
tenant = 'common' # SDK default
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment)
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password or "
"be logged using AzureCLI.")
# base_url for sovereign cloud support. For now only if AzureCLI
if self.credentials.get('base_url') is not None:
self.base_url = self.credentials.get('base_url')
else:
self.base_url = None
# common parameter validation
if self.module.params.get('tags'):
self.validate_tags(self.module.params['tags'])
@ -340,11 +365,10 @@ class AzureRMModuleBase(object):
self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
try:
credentials, subscription_id = get_azure_cli_credentials()
base_url = get_cli_active_cloud().endpoints.resource_manager
self._cloud_environment = get_cli_active_cloud()
return {
'credentials': credentials,
'subscription_id': subscription_id,
'base_url': base_url
'subscription_id': subscription_id
}
except CLIError as err:
self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
@ -641,7 +665,7 @@ class AzureRMModuleBase(object):
self._storage_client = StorageManagementClient(
self.azure_credentials,
self.subscription_id,
base_url=self.base_url,
base_url=self._cloud_environment.endpoints.resource_manager,
api_version='2017-06-01'
)
self._register('Microsoft.Storage')
@ -655,7 +679,7 @@ class AzureRMModuleBase(object):
self._network_client = NetworkManagementClient(
self.azure_credentials,
self.subscription_id,
base_url=self.base_url,
base_url=self._cloud_environment.endpoints.resource_manager,
api_version='2017-06-01'
)
self._register('Microsoft.Network')
@ -669,7 +693,7 @@ class AzureRMModuleBase(object):
self._resource_client = ResourceManagementClient(
self.azure_credentials,
self.subscription_id,
base_url=self.base_url,
base_url=self._cloud_environment.endpoints.resource_manager,
api_version='2017-05-10'
)
return self._resource_client
@ -682,7 +706,7 @@ class AzureRMModuleBase(object):
self._compute_client = ComputeManagementClient(
self.azure_credentials,
self.subscription_id,
base_url=self.base_url,
base_url=self._cloud_environment.endpoints.resource_manager,
api_version='2017-03-30'
)
self._register('Microsoft.Compute')
@ -696,7 +720,7 @@ class AzureRMModuleBase(object):
self._dns_client = DnsManagementClient(
self.azure_credentials,
self.subscription_id,
base_url=self.base_url
base_url=self._cloud_environment.endpoints.resource_manager,
)
self._register('Microsoft.Dns')
return self._dns_client

20
lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py
View file

@ -430,10 +430,10 @@ AZURE_OBJECT_CLASS = 'VirtualMachine'
AZURE_ENUM_MODULES = ['azure.mgmt.compute.models']
def extract_names_from_blob_uri(blob_uri):
def extract_names_from_blob_uri(blob_uri, storage_suffix):
# HACK: ditch this once python SDK supports get by URI
m = re.match('^https://(?P<accountname>[^\.]+)\.blob\.core\.windows\.net/'
'(?P<containername>[^/]+)/(?P<blobname>.+)$', blob_uri)
m = re.match('^https://(?P<accountname>[^\.]+)\.blob\.{0}/'
'(?P<containername>[^/]+)/(?P<blobname>.+)$'.format(storage_suffix), blob_uri)
if not m:
raise Exception("unable to parse blob uri '%s'" % blob_uri)
extracted_names = m.groupdict()
@ -574,9 +574,10 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
if self.storage_account_name:
self.get_storage_account(self.storage_account_name)
requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format(self.storage_account_name,
self.storage_container_name,
self.storage_blob_name)
requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format(self.storage_account_name,
self._cloud_environment.suffixes.storage_endpoint,
self.storage_container_name,
self.storage_blob_name)
disable_ssh_password = not self.ssh_password_enabled
@ -689,8 +690,9 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
storage_account = self.create_default_storage_account()
self.log("storage account:")
self.log(self.serialize_obj(storage_account, 'StorageAccount'), pretty_print=True)
requested_vhd_uri = 'https://{0}.blob.core.windows.net/{1}/{2}'.format(
requested_vhd_uri = 'https://{0}.blob.{1}/{2}/{3}'.format(
storage_account.name,
self._cloud_environment.suffixes.storage_endpoint,
self.storage_container_name,
self.storage_blob_name)
@ -767,7 +769,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
vm_dict['properties']['storageProfile']['osDisk']['name'],
vhd,
vm_dict['properties']['storageProfile']['osDisk']['createOption'],
os_type=vm_dict['properties']['storageProfile']['osDisk']['osType'],
vm_dict['properties']['storageProfile']['osDisk']['osType'],
caching=vm_dict['properties']['storageProfile']['osDisk']['caching']
),
image_reference=ImageReference(
@ -1028,7 +1030,7 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
for uri in vhd_uris:
self.log("Extracting info from blob uri '{0}'".format(uri))
try:
blob_parts = extract_names_from_blob_uri(uri)
blob_parts = extract_names_from_blob_uri(uri, self._cloud_environment.suffixes.storage_endpoint)
except Exception as exc:
self.fail("Error parsing blob URI {0}".format(str(exc)))
storage_account_name = blob_parts['accountname']

13
lib/ansible/utils/module_docs_fragments/azure.py
View file

@ -61,17 +61,22 @@ options:
- Azure tenant ID. Use when authenticating with a Service Principal.
required: false
default: null
cloud_environment:
description:
- For cloud environments other than the US public cloud, the environment name (as defined by Azure Python SDK, eg, C(AzureChinaCloud),
C(AzureUSGovernment)), or a metadata discovery endpoint URL (required for Azure Stack). Can also be set via credential file profile or
the C(AZURE_CLOUD_ENVIRONMENT) environment variable.
default: AzureCloud
requirements:
- "python >= 2.7"
- "azure == 2.0.0rc5"
- "azure >= 2.0.0"
notes:
- For authentication with Azure you can pass parameters, set environment variables or use a profile stored
in ~/.azure/credentials. Authentication is possible using a service principal or Active Directory user.
To authenticate via service principal pass subscription_id, client_id, secret and tenant or set set environment
To authenticate via service principal, pass subscription_id, client_id, secret and tenant or set environment
variables AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID, AZURE_SECRET and AZURE_TENANT.
- To Authentication via Active Directory user pass ad_user and password, or set AZURE_AD_USER and
- To authenticate via Active Directory user, pass ad_user and password, or set AZURE_AD_USER and
AZURE_PASSWORD in the environment.
- "Alternatively, credentials can be stored in ~/.azure/credentials. This is an ini file containing
a [default] section and the following keys: subscription_id, client_id, secret and tenant or