cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

initial cutover to API profiles (#35538)

Browse source 
* hardcoded API profiles in azure_rm_common
* changed azure_rm_securitygroup module to use api_profiles, dynamic models, kwargs on all SDK methods
* changed azure_rm_containerinstance module to use api_profiles, dynamic models, kwargs on all SDK methods
* fixed polling performance issue in azure_rm_securitygroup (default poll interval was 30s)
This commit is contained in:
Matt Davis 2018-02-09 14:26:42 -08:00 committed by GitHub
parent fd33dc6cd1
commit 26f551d1c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 171 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

157
lib/ansible/module_utils/azure_rm_common.py
View file

@ -1,25 +1,11 @@
# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
# Chris Houseknecht, <house@redhat.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
import json
import os
import re
import sys
import types
import copy
import inspect
import traceback
@ -47,7 +33,8 @@ AZURE_COMMON_ARGS = dict(
ad_user=dict(type='str', no_log=True),
password=dict(type='str', no_log=True),
cloud_environment=dict(type='str'),
cert_validation_mode=dict(type='str', choices=['validate', 'ignore'])
cert_validation_mode=dict(type='str', choices=['validate', 'ignore']),
api_profile=dict(type='str', default='latest')
# debug=dict(type='bool', default=False),
)
@ -63,6 +50,31 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
cert_validation_mode='AZURE_CERT_VALIDATION_MODE',
)
# FUTURE: this should come from the SDK or an external location.
# For now, we have to copy from azure-cli
AZURE_API_PROFILES = {
'latest': {
'ContainerInstanceManagementClient': '2018-02-01-preview',
'ComputeManagementClient': dict(
default_api_version='2017-12-01',
resource_skus='2017-09-01',
disks='2017-03-30',
snapshots='2017-03-30',
virtual_machine_run_commands='2017-03-30'
),
'NetworkManagementClient': '2017-11-01',
'ResourceManagementClient': '2017-05-10',
'StorageManagementClient': '2017-10-01'
},
'2017-03-09-profile': {
'ComputeManagementClient': '2016-03-30',
'NetworkManagementClient': '2015-06-15',
'ResourceManagementClient': '2016-02-01',
'StorageManagementClient': '2016-01-01'
}
}
AZURE_TAG_ARGS = dict(
tags=dict(type='dict'),
append_tags=dict(type='bool', default=True),
@ -162,36 +174,36 @@ def format_resource_id(val, subscription_id, namespace, types, resource_group):
type=types,
subscription=subscription_id) if not is_valid_resource_id(val) else val
# FUTURE: either get this from the requirements file (if we can be sure it's always available at runtime)
# or generate the requirements files from this so we only have one source of truth to maintain...
AZURE_PKG_VERSIONS = {
StorageManagementClient.__name__: {
'StorageManagementClient': {
'package_name': 'storage',
'expected_version': '1.5.0',
'installed_version': storage_client_version
'expected_version': '1.5.0'
},
ComputeManagementClient.__name__: {
'ComputeManagementClient': {
'package_name': 'compute',
'expected_version': '2.0.0',
'installed_version': compute_client_version
'expected_version': '2.0.0'
},
NetworkManagementClient.__name__: {
'ContainerInstanceManagementClient': {
'package_name': 'containerinstance',
'expected_version': '0.3.1'
},
'NetworkManagementClient': {
'package_name': 'network',
'expected_version': '1.3.0',
'installed_version': network_client_version
'expected_version': '1.3.0'
},
ResourceManagementClient.__name__: {
'ResourceManagementClient': {
'package_name': 'resource',
'expected_version': '1.1.0',
'installed_version': resource_client_version
'expected_version': '1.1.0'
},
DnsManagementClient.__name__: {
'DnsManagementClient': {
'package_name': 'dns',
'expected_version': '1.0.1',
'installed_version': dns_client_version
'expected_version': '1.0.1'
},
WebSiteManagementClient.__name__: {
'WebSiteManagementClient': {
'package_name': 'web',
'expected_version': '0.32.0',
'installed_version': web_client_version
'expected_version': '0.32.0'
},
} if HAS_AZURE else {}
@ -250,6 +262,7 @@ class AzureRMModuleBase(object):
self._containerservice_client = None
self.check_mode = self.module.check_mode
self.api_profile = self.module.params.get('api_profile')
self.facts_module = facts_module
# self.debug = self.module.params.get('debug')
@ -337,10 +350,15 @@ class AzureRMModuleBase(object):
package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None)
if package_version is not None:
client_name = package_version.get('package_name')
client_version = package_version.get('installed_version')
try:
client_module = importlib.import_module(client_type.__module__)
client_version = client_module.VERSION
except RuntimeError:
# can't get at the module version for some reason, just fail silently...
return
expected_version = package_version.get('expected_version')
if Version(client_version) < Version(expected_version):
self.fail("Installed {0} client version is {1}. The supported version is {2}. Try "
self.fail("Installed azure-mgmt-{0} client version is {1}. The supported version is {2}. Try "
"`pip install ansible[azure]`".format(client_name, client_version, expected_version))
def exec_module(self, **kwargs):
@ -767,18 +785,65 @@ class AzureRMModuleBase(object):
def _validation_ignore_callback(session, global_config, local_config, **kwargs):
session.verify = False
def get_api_profile(self, client_type_name, api_profile_name):
profile_all_clients = AZURE_API_PROFILES.get(api_profile_name)
if not profile_all_clients:
raise KeyError("unknown Azure API profile: {0}".format(api_profile_name))
profile_raw = profile_all_clients.get(client_type_name, None)
if not profile_raw:
self.module.warn("Azure API profile {0} does not define an entry for {1}".format(api_profile_name, client_type_name))
if isinstance(profile_raw, dict):
if not profile_raw.get('default_api_version'):
raise KeyError("Azure API profile {0} does not define 'default_api_version'".format(api_profile_name))
return profile_raw
# wrap basic strings in a dict that just defines the default
return dict(default_api_version=profile_raw)
def get_mgmt_svc_client(self, client_type, base_url=None, api_version=None):
self.log('Getting management service client {0}'.format(client_type.__name__))
self.check_client_version(client_type)
if api_version:
client = client_type(self.azure_credentials,
self.subscription_id,
api_version=api_version,
base_url=base_url)
else:
client = client_type(self.azure_credentials,
self.subscription_id,
base_url=base_url)
client_argspec = inspect.getargspec(client_type.__init__)
client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url)
api_profile_dict = {}
if self.api_profile:
api_profile_dict = self.get_api_profile(client_type.__name__, self.api_profile)
if not base_url:
# most things are resource_manager, don't make everyone specify
base_url = self._cloud_environment.endpoints.resource_manager
# unversioned clients won't accept profile; only send it if necessary
# clients without a version specified in the profile will use the default
if api_profile_dict and 'profile' in client_argspec.args:
client_kwargs['profile'] = api_profile_dict
# If the client doesn't accept api_version, it's unversioned.
# If it does, favor explicitly-specified api_version, fall back to api_profile
if 'api_version' in client_argspec.args:
profile_default_version = api_profile_dict.get('default_api_version', None)
if api_version or profile_default_version:
client_kwargs['api_version'] = api_version or profile_default_version
client = client_type(**client_kwargs)
# FUTURE: remove this once everything exposes models directly (eg, containerinstance)
try:
getattr(client, "models")
except AttributeError:
def _ansible_get_models(self, *arg, **kwarg):
return self._ansible_models
setattr(client, '_ansible_models', importlib.import_module(client_type.__module__).models)
client.models = types.MethodType(_ansible_get_models, client)
# Add user agent for Ansible
client.config.add_user_agent(ANSIBLE_USER_AGENT)

61
lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py
View file

@ -145,14 +145,6 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase
try:
from msrestazure.azure_exceptions import CloudError
from azure.mgmt.containerinstance.models import (ContainerGroup,
Container,
ResourceRequirements,
ResourceRequests,
ImageRegistryCredential,
IpAddress,
Port,
ContainerPort)
from azure.mgmt.containerinstance import ContainerInstanceManagementClient
except ImportError:
# This is handled in azure_rm_common
@ -264,7 +256,8 @@ class AzureRMContainerInstance(AzureRMModuleBase):
self.containers = None
self.results = dict(changed=False, state=dict())
self.mgmt_client = None
self.client = None
self.cgmodels = None
super(AzureRMContainerInstance, self).__init__(derived_arg_spec=self.module_arg_spec,
supports_check_mode=True,
@ -280,8 +273,10 @@ class AzureRMContainerInstance(AzureRMModuleBase):
response = None
results = dict()
self.mgmt_client = self.get_mgmt_svc_client(ContainerInstanceManagementClient,
base_url=self._cloud_environment.endpoints.resource_manager)
self.client = self.get_mgmt_svc_client(ContainerInstanceManagementClient)
# since this client hasn't been upgraded to expose models directly off the OperationClass, fish them out
self.cgmodels = self.client.container_groups.models
resource_group = self.get_resource_group(self.resource_group)
@ -341,9 +336,9 @@ class AzureRMContainerInstance(AzureRMModuleBase):
registry_credentials = None
if self.registry_login_server is not None:
registry_credentials = [ImageRegistryCredential(server=self.registry_login_server,
username=self.registry_username,
password=self.registry_password)]
registry_credentials = [self.cgmodels.ImageRegistryCredential(server=self.registry_login_server,
username=self.registry_username,
password=self.registry_password)]
ip_address = None
@ -352,8 +347,8 @@ class AzureRMContainerInstance(AzureRMModuleBase):
if self.ports:
ports = []
for port in self.ports:
ports.append(Port(port=port, protocol="TCP"))
ip_address = IpAddress(ports=ports, ip=self.ip_address)
ports.append(self.cgmodels.Port(port=port, protocol="TCP"))
ip_address = self.cgmodels.IpAddress(ports=ports, ip=self.ip_address)
containers = []
@ -367,22 +362,26 @@ class AzureRMContainerInstance(AzureRMModuleBase):
port_list = container_def.get("ports")
if port_list:
for port in port_list:
ports.append(ContainerPort(port))
ports.append(self.cgmodels.ContainerPort(port=port))
containers.append(Container(name=name,
image=image,
resources=ResourceRequirements(ResourceRequests(memory_in_gb=memory, cpu=cpu)),
ports=ports))
containers.append(self.cgmodels.Container(name=name,
image=image,
resources=self.cgmodels.ResourceRequirements(
requests=self.cgmodels.ResourceRequests(memory_in_gb=memory, cpu=cpu)
),
ports=ports))
parameters = ContainerGroup(location=self.location,
containers=containers,
image_registry_credentials=registry_credentials,
restart_policy=None,
ip_address=ip_address,
os_type=self.os_type,
volumes=None)
parameters = self.cgmodels.ContainerGroup(location=self.location,
containers=containers,
image_registry_credentials=registry_credentials,
restart_policy=None,
ip_address=ip_address,
os_type=self.os_type,
volumes=None)
response = self.mgmt_client.container_groups.create_or_update(self.resource_group, self.name, parameters)
response = self.client.container_groups.create_or_update(resource_group_name=self.resource_group,
container_group_name=self.name,
container_group=parameters)
return response.as_dict()
@ -393,7 +392,7 @@ class AzureRMContainerInstance(AzureRMModuleBase):
:return: True
'''
self.log("Deleting the container instance {0}".format(self.name))
response = self.mgmt_client.container_groups.delete(self.resource_group, self.name)
response = self.client.container_groups.delete(resource_group_name=self.resource_group, container_group_name=self.name)
return True
def get_containerinstance(self):
@ -405,7 +404,7 @@ class AzureRMContainerInstance(AzureRMModuleBase):
self.log("Checking if the container instance {0} is present".format(self.name))
found = False
try:
response = self.mgmt_client.container_groups.get(self.resource_group, self.name)
response = self.client.container_groups.get(resource_group_name=self.resource_group, container_group_name=self.name)
found = True
self.log("Response : {0}".format(response))
self.log("Container instance : {0} found".format(response.name))

32
lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py
View file

@ -334,6 +334,7 @@ state:
try:
from msrestazure.azure_exceptions import CloudError
from azure.mgmt.network import NetworkManagementClient
except ImportError:
# This is handled in azure_rm_common
pass
@ -369,7 +370,7 @@ def validate_rule(self, rule, rule_type=None):
if not rule.get('access'):
rule['access'] = 'Allow'
access_names = [member.value for member in self.network_models.SecurityRuleAccess]
access_names = [member.value for member in self.nsg_models.SecurityRuleAccess]
if rule['access'] not in access_names:
raise Exception("Rule access must be one of [{0}]".format(', '.join(access_names)))
@ -382,14 +383,14 @@ def validate_rule(self, rule, rule_type=None):
if not rule.get('protocol'):
rule['protocol'] = '*'
protocol_names = [member.value for member in self.network_models.SecurityRuleProtocol]
protocol_names = [member.value for member in self.nsg_models.SecurityRuleProtocol]
if rule['protocol'] not in protocol_names:
raise Exception("Rule protocol must be one of [{0}]".format(', '.join(protocol_names)))
if not rule.get('direction'):
rule['direction'] = 'Inbound'
direction_names = [member.value for member in self.network_models.SecurityRuleDirection]
direction_names = [member.value for member in self.nsg_models.SecurityRuleDirection]
if rule['direction'] not in direction_names:
raise Exception("Rule direction must be one of [{0}]".format(', '.join(direction_names)))
@ -439,7 +440,7 @@ def create_rule_instance(self, rule):
:param rule: dict
:return: SecurityRule
'''
return self.network_models.SecurityRule(
return self.nsg_models.SecurityRule(
protocol=rule['protocol'],
source_address_prefix=rule['source_address_prefix'],
destination_address_prefix=rule['destination_address_prefix'],
@ -535,6 +536,8 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
self.rules = None
self.state = None
self.tags = None
self.client = None # type: azure.mgmt.network.NetworkManagementClient
self.nsg_models = None # type: azure.mgmt.network.models
self.results = dict(
changed=False,
@ -545,6 +548,11 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
supports_check_mode=True)
def exec_module(self, **kwargs):
self.client = self.get_mgmt_svc_client(NetworkManagementClient)
# tighten up poll interval for security groups; default 30s is an eternity
# this value is still overridden by the response Retry-After header (which is set on the initial operation response to 10s)
self.client.config.long_running_operation_timeout = 3
self.nsg_models = self.client.network_security_groups.models
for key in list(self.module_arg_spec.keys()) + ['tags']:
setattr(self, key, kwargs[key])
@ -572,7 +580,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
self.fail("Error validating default rule {0} - {1}".format(rule, str(exc)))
try:
nsg = self.network_client.network_security_groups.get(self.resource_group, self.name)
nsg = self.client.network_security_groups.get(self.resource_group, self.name)
results = create_network_security_group_dict(nsg)
self.log("Found security group:")
self.log(results, pretty_print=True)
@ -582,7 +590,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
elif self.state == 'absent':
self.log("CHANGED: security group found but state is 'absent'")
changed = True
except CloudError:
except CloudError: # TODO: actually check for ResourceMissingError
if self.state == 'present':
self.log("CHANGED: security group not found and state is 'present'")
changed = True
@ -640,7 +648,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
self.results['changed'] = changed
self.results['state'] = results
if not self.check_mode:
if not self.check_mode and changed:
self.results['state'] = self.create_or_update(results)
elif self.state == 'present' and changed:
@ -681,7 +689,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
return self.results
def create_or_update(self, results):
parameters = self.network_models.NetworkSecurityGroup()
parameters = self.nsg_models.NetworkSecurityGroup()
if results.get('rules'):
parameters.security_rules = []
for rule in results.get('rules'):
@ -694,9 +702,9 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
parameters.location = results.get('location')
try:
poller = self.network_client.network_security_groups.create_or_update(self.resource_group,
self.name,
parameters)
poller = self.client.network_security_groups.create_or_update(resource_group_name=self.resource_group,
network_security_group_name=self.name,
parameters=parameters)
result = self.get_poller_result(poller)
except CloudError as exc:
self.fail("Error creating/updating security group {0} - {1}".format(self.name, str(exc)))
@ -704,7 +712,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
def delete(self):
try:
poller = self.network_client.network_security_groups.delete(self.resource_group, self.name)
poller = self.client.network_security_groups.delete(resource_group_name=self.resource_group, network_security_group_name=self.name)
result = self.get_poller_result(poller)
except CloudError as exc:
raise Exception("Error deleting security group {0} - {1}".format(self.name, str(exc)))

8
lib/ansible/utils/module_docs_fragments/azure.py
View file

@ -90,6 +90,14 @@ options:
- env
default: auto
version_added: 2.5
api_profile:
description:
- Selects an API profile to use when communicating with Azure services. Default value of C(latest) is appropriate for public clouds;
future values will allow use with Azure Stack.
choices:
- latest
default: latest
version_added: 2.5
requirements:
- "python >= 2.7"
- "azure >= 2.0.0"

2
packaging/requirements/requirements-azure.txt
View file

@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33
azure-mgmt-containerservice>=2.0.0,<3.0.0
azure-mgmt-containerregistry>=1.0.1
azure-mgmt-rdbms>=0.2.0rc1,<0.3.0
azure-mgmt-containerinstance>=0.2.0,<0.3.0
azure-mgmt-containerinstance>=0.3.1

2
test/runner/requirements/integration.cloud.azure.txt
View file

@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33
azure-mgmt-containerservice>=2.0.0,<3.0.0
azure-mgmt-containerregistry>=1.0.1
azure-mgmt-rdbms>=0.2.0rc1,<0.3.0
azure-mgmt-containerinstance>=0.2.0,<0.3.0
azure-mgmt-containerinstance>=0.3.1