Add support for Azure 2.0.0 (#27920)

* Adapt azure_rm_resource_group to azure 2.0.0 + azure Cli support

* Fix exceptions in Azure ARM plugins

* update azure_rm_networkinterface documention to reflect required params

* change state param to not required for docs in azure_rm_subnet

* fix import to reflect azure==2.0.0 changes

* add aliases and fix docs for azure_rm_storageblob

* add resource_group_name alias to azure_rm_storageaccount_facts

* fix import bug due to change in azure==2.0.0

* fix args bug and enum modules issue

* update docs to reflect azure==2.0.0

* pin management clients to a specific api_version

* update docs to reflect the new azure-ansible-base python package

* add fallback for older api resource group listing

* rework azure dependencies installation

* refactor path joining to a cross-plat solution

docs/docsite/rst/guide_azure.rst

@@ -7,14 +7,19 @@ and orchestrate infrastructure on the Microsoft Azure Cloud.
 Requirements
 ------------
 
-Using the Azure Resource Manager modules requires having `Azure Python SDK <https://github.com/Azure/azure-sdk-for-python>`_
-installed on the host running Ansible. You will need to have == v2.0.0RC5 installed. The simplest way to install the
-SDK is via pip:
+Using the Azure Resource Manager modules requires having specific Azure SDK modules
+installed on the host running Ansible.
 
 .. code-block:: bash
 
-    $ pip install "azure==2.0.0rc5"
+    $ pip install ansible[azure]
 
+If you are running Ansible from source, you can install the dependencies from the
+root directory of the Ansible repo.
+
+.. code-block:: bash
+
+    $ pip install .[azure]
 
 Authenticating with Azure
 -------------------------

lib/ansible/module_utils/azure_rm_common.py

@@ -31,6 +31,7 @@ from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six.moves import configparser
 
 AZURE_COMMON_ARGS = dict(
+    cli_default_profile=dict(type='bool'),
     profile=dict(type='str'),
     subscription_id=dict(type='str', no_log=True),
     client_id=dict(type='str', no_log=True),
@@ -42,6 +43,7 @@ AZURE_COMMON_ARGS = dict(
 )
 
 AZURE_CREDENTIAL_ENV_MAPPING = dict(
+    cli_default_profile='AZURE_CLI_DEFAULT_PROFILE',
     profile='AZURE_PROFILE',
     subscription_id='AZURE_SUBSCRIPTION_ID',
     client_id='AZURE_CLIENT_ID',
@@ -70,6 +72,7 @@ AZURE_FAILED_STATE = "Failed"
 
 HAS_AZURE = True
 HAS_AZURE_EXC = None
+HAS_AZURE_CLI_CORE = True
 
 HAS_MSRESTAZURE = True
 HAS_MSRESTAZURE_EXC = None
@@ -91,16 +94,24 @@ try:
     from azure.mgmt.storage.version import VERSION as storage_client_version
     from azure.mgmt.compute.version import VERSION as compute_client_version
     from azure.mgmt.resource.version import VERSION as resource_client_version
-    from azure.mgmt.network.network_management_client import NetworkManagementClient
-    from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient
-    from azure.mgmt.storage.storage_management_client import StorageManagementClient
-    from azure.mgmt.compute.compute_management_client import ComputeManagementClient
+    from azure.mgmt.network import NetworkManagementClient
+    from azure.mgmt.resource.resources import ResourceManagementClient
+    from azure.mgmt.storage import StorageManagementClient
+    from azure.mgmt.compute import ComputeManagementClient
     from azure.storage.cloudstorageaccount import CloudStorageAccount
 except ImportError as exc:
     HAS_AZURE_EXC = exc
     HAS_AZURE = False
 
 
+try:
+    from azure.cli.core.util import CLIError
+    from azure.common.credentials import get_azure_cli_credentials, get_cli_profile
+    from azure.common.cloud import get_cli_active_cloud
+except ImportError:
+    HAS_AZURE_CLI_CORE = False
+
+
 def azure_id_to_dict(id):
     pieces = re.sub(r'^\/', '', id).split('/')
     result = {}
@@ -112,13 +123,13 @@ def azure_id_to_dict(id):
 
 
 AZURE_EXPECTED_VERSIONS = dict(
-    storage_client_version="0.30.0rc5",
-    compute_client_version="0.30.0rc5",
-    network_client_version="0.30.0rc5",
-    resource_client_version="0.30.0rc5"
+    storage_client_version="1.0.0",
+    compute_client_version="1.0.0",
+    network_client_version="1.0.0",
+    resource_client_version="1.1.0"
 )
 
-AZURE_MIN_RELEASE = '2.0.0rc5'
+AZURE_MIN_RELEASE = '2.0.0'
 
 
 class AzureRMModuleBase(object):
@@ -171,7 +182,7 @@ class AzureRMModuleBase(object):
         self.credentials = self._get_credentials(self.module.params)
         if not self.credentials:
             self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
-                      "or define a profile in ~/.azure/credentials.")
+                      "or define a profile in ~/.azure/credentials or be logged using AzureCLI.")
 
         if self.credentials.get('subscription_id', None) is None:
             self.fail("Credentials did not include a subscription_id value.")
@@ -192,7 +203,14 @@ class AzureRMModuleBase(object):
                 self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
         else:
             self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
-                      "Credentials must include client_id, secret and tenant or ad_user and password.")
+                      "Credentials must include client_id, secret and tenant or ad_user and password or "
+                      "be logged using AzureCLI.")
+
+        # base_url for sovereign cloud support. For now only if AzureCLI
+        if self.credentials.get('base_url') is not None:
+            self.base_url = self.credentials.get('base_url')
+        else:
+            self.base_url = None
 
         # common parameter validation
         if self.module.params.get('tags'):
@@ -313,6 +331,20 @@ class AzureRMModuleBase(object):
         except Exception as exc:
             self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc)))
 
+    def _get_azure_cli_profile(self):
+        if not HAS_AZURE_CLI_CORE:
+            self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
+        try:
+            credentials, subscription_id = get_azure_cli_credentials()
+            base_url = get_cli_active_cloud().endpoints.resource_manager
+            return {
+                'credentials': credentials,
+                'subscription_id': subscription_id,
+                'base_url': base_url
+            }
+        except CLIError as err:
+            self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
+
     def _get_profile(self, profile="default"):
         path = expanduser("~/.azure/credentials")
         try:
@@ -338,6 +370,9 @@ class AzureRMModuleBase(object):
         for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
             env_credentials[attribute] = os.environ.get(env_variable, None)
 
+        if (env_credentials['cli_default_profile'] or '').lower() in ["true", "yes", "1"]:
+            return self._get_azure_cli_profile()
+
         if env_credentials['profile']:
             credentials = self._get_profile(env_credentials['profile'])
             return credentials
@@ -357,6 +392,10 @@ class AzureRMModuleBase(object):
         for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
             arg_credentials[attribute] = params.get(attribute, None)
 
+        if arg_credentials['cli_default_profile']:
+            self.log('Retrieving credentials from Azure CLI current profile')
+            return self._get_azure_cli_profile()
+
         # try module params
         if arg_credentials['profile'] is not None:
             self.log('Retrieving credentials with profile parameter.')
@@ -595,7 +634,12 @@ class AzureRMModuleBase(object):
         self.log('Getting storage client...')
         if not self._storage_client:
             self.check_client_version('storage', storage_client_version, AZURE_EXPECTED_VERSIONS['storage_client_version'])
-            self._storage_client = StorageManagementClient(self.azure_credentials, self.subscription_id)
+            self._storage_client = StorageManagementClient(
+                self.azure_credentials,
+                self.subscription_id,
+                base_url=self.base_url,
+                api_version='2017-06-01'
+            )
             self._register('Microsoft.Storage')
         return self._storage_client
 
@@ -604,7 +648,12 @@ class AzureRMModuleBase(object):
         self.log('Getting network client')
         if not self._network_client:
             self.check_client_version('network', network_client_version, AZURE_EXPECTED_VERSIONS['network_client_version'])
-            self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id)
+            self._network_client = NetworkManagementClient(
+                self.azure_credentials,
+                self.subscription_id,
+                base_url=self.base_url,
+                api_version='2017-06-01'
+            )
             self._register('Microsoft.Network')
         return self._network_client
 
@@ -613,7 +662,12 @@ class AzureRMModuleBase(object):
         self.log('Getting resource manager client')
         if not self._resource_client:
             self.check_client_version('resource', resource_client_version, AZURE_EXPECTED_VERSIONS['resource_client_version'])
-            self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id)
+            self._resource_client = ResourceManagementClient(
+                self.azure_credentials,
+                self.subscription_id,
+                base_url=self.base_url,
+                api_version='2017-05-10'
+            )
         return self._resource_client
 
     @property
@@ -621,6 +675,11 @@ class AzureRMModuleBase(object):
         self.log('Getting compute client')
         if not self._compute_client:
             self.check_client_version('compute', compute_client_version, AZURE_EXPECTED_VERSIONS['compute_client_version'])
-            self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id)
+            self._compute_client = ComputeManagementClient(
+                self.azure_credentials,
+                self.subscription_id,
+                base_url=self.base_url,
+                api_version='2017-03-30'
+            )
             self._register('Microsoft.Compute')
         return self._compute_client

lib/ansible/modules/cloud/azure/azure_rm_deployment.py

@@ -371,6 +371,13 @@ import time
 
 try:
     from azure.common.credentials import ServicePrincipalCredentials
+    import time
+    import yaml
+except ImportError as exc:
+    IMPORT_ERROR = "Error importing module prerequisites: %s" % exc
+
+try:
+    from itertools import chain
     from azure.common.exceptions import CloudError
     from azure.mgmt.resource.resources.models import (DeploymentProperties,
                                                       ParametersLink,

lib/ansible/modules/cloud/azure/azure_rm_networkinterface.py

@@ -57,7 +57,7 @@ options:
               when creating a network interface.
         aliases:
             - virtual_network
-        required: false
+        required: true
         default: null
     subnet_name:
         description:
@@ -65,7 +65,7 @@ options:
               interface
         aliases:
             - subnet
-        required: false
+        required: true
         default: null
     os_type:
         description:

lib/ansible/modules/cloud/azure/azure_rm_resourcegroup.py

@@ -233,9 +233,12 @@ class AzureRMResourceGroup(AzureRMModuleBase):
     def resources_exist(self):
         found = False
         try:
+            response = self.rm_client.resources.list_by_resource_group(self.name)
+        except AttributeError:
             response = self.rm_client.resource_groups.list_resources(self.name)
         except Exception as exc:
             self.fail("Error checking for resource existence in {0} - {1}".format(self.name, str(exc)))
+
         for item in response:
             found = True
             break

lib/ansible/modules/cloud/azure/azure_rm_resourcegroup_facts.py

@@ -81,7 +81,6 @@ azure_resourcegroups:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
 except:
     # This is handled in azure_rm_common
     pass
@@ -144,7 +143,7 @@ class AzureRMResourceGroupFacts(AzureRMModuleBase):
         self.log('List all items')
         try:
             response = self.rm_client.resource_groups.list()
-        except AzureHttpError as exc:
+        except CloudError as exc:
             self.fail("Failed to list all items - {1}".format(str(exc)))
 
         results = []

lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py

@@ -328,11 +328,12 @@ state:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureHttpError
     from azure.mgmt.network.models import NetworkSecurityGroup, SecurityRule
-    from azure.mgmt.network.models.network_management_client_enums import (SecurityRuleAccess,
-                                                                           SecurityRuleDirection,
-                                                                           SecurityRuleProtocol)
+    from azure.mgmt.network.models import (
+        SecurityRuleAccess,
+        SecurityRuleDirection,
+        SecurityRuleProtocol
+    )
 except ImportError:
     # This is handled in azure_rm_common
     pass
@@ -694,7 +695,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
                                                                                   self.name,
                                                                                   parameters)
             result = self.get_poller_result(poller)
-        except AzureHttpError as exc:
+        except CloudError as exc:
             self.fail("Error creating/updating security group {0} - {1}".format(self.name, str(exc)))
         return create_network_security_group_dict(result)
 
@@ -702,7 +703,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
         try:
             poller = self.network_client.network_security_groups.delete(self.resource_group, self.name)
             result = self.get_poller_result(poller)
-        except AzureHttpError as exc:
+        except CloudError as exc:
             raise Exception("Error deleting security group {0} - {1}".format(self.name, str(exc)))
         return result
 

lib/ansible/modules/cloud/azure/azure_rm_securitygroup_facts.py

@@ -194,7 +194,6 @@ azure_securitygroups:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
 except:
     # This is handled in azure_rm_common
     pass

lib/ansible/modules/cloud/azure/azure_rm_storageaccount.py

@@ -26,6 +26,8 @@ options:
         description:
             - Name of the resource group to use.
         required: true
+        aliases:
+            - resource_group_name
     name:
         description:
             - Name of the storage account to update or create.
@@ -141,8 +143,8 @@ state:
 try:
     from msrestazure.azure_exceptions import CloudError
     from azure.storage.cloudstorageaccount import CloudStorageAccount
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
-    from azure.mgmt.storage.models.storage_management_client_enums import ProvisioningState, SkuName, SkuTier, Kind
+    from azure.common import AzureMissingResourceHttpError
+    from azure.mgmt.storage.models import ProvisioningState, SkuName, SkuTier, Kind
     from azure.mgmt.storage.models import StorageAccountUpdateParameters, CustomDomain, \
                                           StorageAccountCreateParameters, Sku
 except ImportError:
@@ -161,7 +163,7 @@ class AzureRMStorageAccount(AzureRMModuleBase):
             custom_domain=dict(type='dict'),
             location=dict(type='str'),
             name=dict(type='str', required=True),
-            resource_group=dict(required=True, type='str'),
+            resource_group=dict(required=True, type='str', aliases=['resource_group_name']),
             state=dict(default='present', choices=['present', 'absent']),
             force=dict(type='bool', default=False),
             tags=dict(type='dict'),
@@ -237,7 +239,7 @@ class AzureRMStorageAccount(AzureRMModuleBase):
         self.log('Checking name availability for {0}'.format(self.name))
         try:
             response = self.storage_client.storage_accounts.check_name_availability(self.name)
-        except AzureHttpError as e:
+        except CloudError as e:
             self.log('Error attempting to validate name.')
             self.fail("Error checking name availability: {0}".format(str(e)))
         if not response.name_available:
@@ -386,7 +388,7 @@ class AzureRMStorageAccount(AzureRMModuleBase):
         try:
             poller = self.storage_client.storage_accounts.create(self.resource_group, self.name, parameters)
             self.get_poller_result(poller)
-        except AzureHttpError as e:
+        except CloudError as e:
             self.log('Error creating storage account.')
             self.fail("Failed to create account: {0}".format(str(e)))
         # the poller doesn't actually return anything
@@ -404,7 +406,7 @@ class AzureRMStorageAccount(AzureRMModuleBase):
                 status = self.storage_client.storage_accounts.delete(self.resource_group, self.name)
                 self.log("delete status: ")
                 self.log(str(status))
-            except AzureHttpError as e:
+            except CloudError as e:
                 self.fail("Failed to delete the account: {0}".format(str(e)))
         return True
 

lib/ansible/modules/cloud/azure/azure_rm_storageaccount_facts.py

@@ -36,6 +36,8 @@ options:
             - Limit results to a resource group. Required when filtering by name.
         required: false
         default: null
+        aliases:
+            - resource_group_name
     tags:
         description:
             - Limit results by providing a list of tags. Format tags as 'key' or 'key:value'.
@@ -97,7 +99,6 @@ azure_storageaccounts:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
 except:
     # This is handled in azure_rm_common
     pass
@@ -113,7 +114,7 @@ class AzureRMStorageAccountFacts(AzureRMModuleBase):
 
         self.module_arg_spec = dict(
             name=dict(type='str'),
-            resource_group=dict(type='str'),
+            resource_group=dict(type='str', aliases=['resource_group_name']),
             tags=dict(type='list'),
         )
 

lib/ansible/modules/cloud/azure/azure_rm_storageblob.py

@@ -29,6 +29,7 @@ options:
         required: true
         aliases:
             - account_name
+            - storage_account
     blob:
         description:
             - Name of a blob object within the container.
@@ -89,6 +90,8 @@ options:
         description:
             - Name of the resource group to use.
         required: true
+        aliases:
+            - resource_group_name
     src:
         description:
             - Source file path. Use with state 'present' to upload a blob.
@@ -207,12 +210,12 @@ class AzureRMStorageBlob(AzureRMModuleBase):
     def __init__(self):
 
         self.module_arg_spec = dict(
-            storage_account_name=dict(required=True, type='str', aliases=['account_name']),
+            storage_account_name=dict(required=True, type='str', aliases=['account_name', 'storage_account']),
             blob=dict(type='str', aliases=['blob_name']),
             container=dict(required=True, type='str', aliases=['container_name']),
             dest=dict(type='str'),
             force=dict(type='bool', default=False),
-            resource_group=dict(required=True, type='str'),
+            resource_group=dict(required=True, type='str', aliases=['resource_group_name']),
             src=dict(type='str'),
             state=dict(type='str', default='present', choices=['absent', 'present']),
             public_access=dict(type='str', choices=['container', 'blob']),

lib/ansible/modules/cloud/azure/azure_rm_subnet.py

@@ -50,7 +50,7 @@ options:
         description:
             - Assert the state of the subnet. Use 'present' to create or update a subnet and
               'absent' to delete a subnet.
-        required: true
+        required: false
         default: present
         choices:
             - absent

lib/ansible/modules/cloud/azure/azure_rm_virtualmachine.py

@@ -416,8 +416,8 @@ try:
     from azure.mgmt.network.models import PublicIPAddress, NetworkSecurityGroup, NetworkInterface, \
                                           NetworkInterfaceIPConfiguration, Subnet
     from azure.mgmt.storage.models import StorageAccountCreateParameters, Sku
-    from azure.mgmt.storage.models.storage_management_client_enums import Kind, SkuTier, SkuName
-    from azure.mgmt.compute.models.compute_management_client_enums import VirtualMachineSizeTypes, DiskCreateOptionTypes
+    from azure.mgmt.storage.models import Kind, SkuTier, SkuName
+    from azure.mgmt.compute.models import VirtualMachineSizeTypes, DiskCreateOptionTypes
 except ImportError:
     # This is handled in azure_rm_common
     pass
@@ -427,7 +427,7 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase, azure_id_to_
 
 AZURE_OBJECT_CLASS = 'VirtualMachine'
 
-AZURE_ENUM_MODULES = ['azure.mgmt.compute.models.compute_management_client_enums']
+AZURE_ENUM_MODULES = ['azure.mgmt.compute.models']
 
 
 def extract_names_from_blob_uri(blob_uri):
@@ -711,9 +711,9 @@ class AzureRMVirtualMachine(AzureRMModuleBase):
                         ),
                         storage_profile=StorageProfile(
                             os_disk=OSDisk(
-                                self.storage_blob_name,
-                                vhd,
-                                DiskCreateOptionTypes.from_image,
+                                name=self.storage_blob_name,
+                                vhd=vhd,
+                                create_option=DiskCreateOptionTypes.from_image,
                                 caching=self.os_disk_caching,
                             ),
                             image_reference=ImageReference(

lib/ansible/modules/cloud/azure/azure_rm_virtualmachineimage_facts.py

@@ -103,7 +103,6 @@ azure_vmimages:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
 except:
     # This is handled in azure_rm_common
     pass

lib/ansible/modules/cloud/azure/azure_rm_virtualnetwork_facts.py

@@ -92,7 +92,6 @@ azure_virtualnetworks:
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
 except:
     # This is handled in azure_rm_common
     pass
@@ -157,7 +156,7 @@ class AzureRMNetworkInterfaceFacts(AzureRMModuleBase):
         self.log('List items for resource group')
         try:
             response = self.network_client.virtual_networks.list(self.resource_group)
-        except AzureHttpError as exc:
+        except CloudError as exc:
             self.fail("Failed to list for resource group {0} - {1}".format(self.resource_group, str(exc)))
 
         results = []
@@ -170,7 +169,7 @@ class AzureRMNetworkInterfaceFacts(AzureRMModuleBase):
         self.log('List all for items')
         try:
             response = self.network_client.virtual_networks.list_all()
-        except AzureHttpError as exc:
+        except CloudError as exc:
             self.fail("Failed to list all items - {0}".format(str(exc)))
 
         results = []

packaging/requirements/requirements-azure.txt

@@ -0,0 +1,12 @@
+azure-mgmt-compute~=2.0.0
+azure-mgmt-network~=1.3.0
+azure-mgmt-storage~=1.2.0
+azure-mgmt-resource~=1.1.0
+azure-storage~=0.35.1
+azure-cli-core~=2.0.12
+msrestazure~=0.4.11
+azure-mgmt-dns~=1.0.1
+azure-mgmt-keyvault~=0.40.0
+azure-mgmt-batch~=4.1.0
+azure-mgmt-sql~=0.7.1
+azure-mgmt-web~=0.32.0

setup.py

@@ -2,6 +2,7 @@
 import json
 import os
 import os.path
+import re
 import sys
 from collections import defaultdict
 from distutils.command.build_scripts import build_scripts as BuildScripts
@@ -147,6 +148,15 @@ if crypto_backend:
     install_requirements = [r for r in install_requirements if not (r.lower().startswith('pycrypto') or r.lower().startswith('cryptography'))]
     install_requirements.append(crypto_backend)
 
+# specify any extra requirements for installation
+extra_requirements = dict()
+extra_requirements_dir = 'packaging/requirements'
+for extra_requirements_filename in os.listdir(extra_requirements_dir):
+    filename_match = re.search(r'^requirements-(\w*).txt$', extra_requirements_filename)
+    if filename_match:
+        with open(os.path.join(extra_requirements_dir, extra_requirements_filename)) as extra_requirements_file:
+            extra_requirements[filename_match.group(1)] = extra_requirements_file.read().splitlines()
+
 
 setup(
     # Use the distutils SDist so that symlinks are not expanded
@@ -210,4 +220,5 @@ setup(
         'bin/ansible-vault',
     ],
     data_files=[],
+    extras_require=extra_requirements
 )