Add Azure support to ansible-test. (#28092)

* Initial Azure cloud testing support.
* Add missing Azure requirements.
* Fix test handling of setup and requirements.
* Update Azure cloud plugin.
* Add setup_azure role for integration tests.
* Update minimal Azure integration test sample.
This commit is contained in:
Matt Clay 2017-08-11 17:33:39 -07:00 committed by GitHub
parent c27ebfc368
commit caf8bbf3bd
11 changed files with 261 additions and 8 deletions

View file

@ -1,3 +1,5 @@
packaging
requests[security]
azure-mgmt-compute>=2.0.0,<3
azure-mgmt-network>=1.3.0,<2
azure-mgmt-storage>=1.2.0,<2

View file

@ -0,0 +1,31 @@
# This is the configuration template for ansible-test Azure integration tests.
#
# You do not need this template if you are:
#
# 1) Running integration tests without using ansible-test.
# 2) Using the automatically provisioned Azure credentials in ansible-test.
#
# If you do not want to use the automatically provisioned temporary Azure credentials,
# fill in the values below and save this file without the .template extension.
# This will cause ansible-test to use the given configuration instead of temporary credentials.
#
# NOTE: Automatic provisioning of Azure credentials requires one of:
# 1) ansible-core-ci API key in ~/.ansible-core-ci.key
# 2) Sherlock URL (including API key) in ~/.ansible-sherlock-ci.cfg
# Provide either Service Principal or Active Directory credentials below.
# Service Principal
AZURE_CLIENT_ID=
AZURE_SECRET=
AZURE_SUBSCRIPTION_ID=
AZURE_TENANT=
# Active Directory
AZURE_AD_USER=
AZURE_PASSWORD=
AZURE_SUBSCRIPTION_ID=
# Resource Groups
RESOURCE_GROUP=
RESOURCE_GROUP_SECONDARY=

View file

@ -50,3 +50,6 @@ overridden_in_parent=2000
[amazon]
localhost ansible_ssh_host=127.0.0.1 ansible_connection=local
[azure]
localhost ansible_ssh_host=127.0.0.1 ansible_connection=local

View file

@ -0,0 +1,2 @@
cloud/azure
destructive

View file

@ -0,0 +1,2 @@
dependencies:
- setup_azure

View file

@ -0,0 +1,7 @@
- block:
- name: create a virtual network
azure_rm_virtualnetwork:
resource_group: '{{ resource_group }}'
name: test
address_prefixes_cidr:
- "10.1.0.0/16"

View file

@ -0,0 +1,2 @@
- pip:
requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt'

View file

@ -347,9 +347,20 @@ class PathMapper(object):
if path.startswith('packaging/'):
if path.startswith('packaging/requirements/'):
return {
'integration': 'ansible',
}
if name.startswith('requirements-') and ext == '.txt':
component = name.split('-', 1)[1]
candidates = (
'cloud/%s/' % component,
)
for candidate in candidates:
if candidate in self.integration_targets_by_alias:
return {
'integration': candidate,
}
return all_tests(self.args) # broad impact, run all tests
return minimal
@ -467,9 +478,7 @@ class PathMapper(object):
return all_tests(self.args) # test infrastructure, run all tests
if path == 'setup.py':
return {
'integration': 'ansible',
}
return all_tests(self.args) # broad impact, run all tests
if path == '.yamllint':
return {

View file

@ -0,0 +1,194 @@
"""Azure plugin for integration tests."""
from __future__ import absolute_import, print_function
import os
from lib.util import (
ApplicationError,
display,
is_shippable,
)
from lib.cloud import (
CloudProvider,
CloudEnvironment,
)
from lib.http import (
HttpClient,
urlparse,
urlunparse,
parse_qs,
)
from lib.core_ci import (
AnsibleCoreCI,
)
class AzureCloudProvider(CloudProvider):
"""Azure cloud provider plugin. Sets up cloud resources before delegation."""
SHERLOCK_CONFIG_PATH = os.path.expanduser('~/.ansible-sherlock-ci.cfg')
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
if os.path.isfile(self.config_static_path):
return
aci = self._create_ansible_core_ci()
if os.path.isfile(aci.ci_key):
return
if os.path.isfile(self.SHERLOCK_CONFIG_PATH):
return
if is_shippable():
return
super(AzureCloudProvider, self).filter(targets, exclude)
def setup(self):
"""Setup the cloud resource before delegation and register a cleanup callback."""
super(AzureCloudProvider, self).setup()
if not self._use_static_config():
self._setup_dynamic()
get_config(self.config_path) # check required variables
def _setup_dynamic(self):
"""Request Azure credentials through Sherlock."""
display.info('Provisioning %s cloud environment.' % self.platform, verbosity=1)
config = self._read_config_template()
response = {}
if os.path.isfile(self.SHERLOCK_CONFIG_PATH):
with open(self.SHERLOCK_CONFIG_PATH, 'r') as sherlock_fd:
sherlock_uri = sherlock_fd.readline().strip() + '&rgcount=2'
parts = urlparse(sherlock_uri)
query_string = parse_qs(parts.query)
base_uri = urlunparse(parts[:4] + ('', ''))
if 'code' not in query_string:
example_uri = 'https://example.azurewebsites.net/api/sandbox-provisioning'
raise ApplicationError('The Sherlock URI must include the API key in the query string. Example: %s?code=xxx' % example_uri)
display.info('Initializing azure/sherlock from: %s' % base_uri, verbosity=1)
http = HttpClient(self.args)
result = http.get(sherlock_uri)
display.info('Started azure/sherlock from: %s' % base_uri, verbosity=1)
if not self.args.explain:
response = result.json()
else:
aci = self._create_ansible_core_ci()
aci_result = aci.start()
if not self.args.explain:
response = aci_result['azure']
if not self.args.explain:
values = dict(
AZURE_CLIENT_ID=response['clientId'],
AZURE_SECRET=response['clientSecret'],
AZURE_SUBSCRIPTION_ID=response['subscriptionId'],
AZURE_TENANT=response['tenantId'],
RESOURCE_GROUP=response['resourceGroupNames'][0],
RESOURCE_GROUP_SECONDARY=response['resourceGroupNames'][1],
)
config = '\n'.join('%s: %s' % (key, values[key]) for key in sorted(values))
self._write_config(config)
def _create_ansible_core_ci(self):
"""
:rtype: AnsibleCoreCI
"""
return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage)
class AzureCloudEnvironment(CloudEnvironment):
"""Azure cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
"""
:type env: dict[str, str]
:type cmd: list[str]
"""
config = get_config(self.config_path)
cmd.append('-e')
cmd.append('resource_prefix=%s' % self.resource_prefix)
cmd.append('-e')
cmd.append('resource_group=%s' % config['RESOURCE_GROUP'])
cmd.append('-e')
cmd.append('resource_group_secondary=%s' % config['RESOURCE_GROUP_SECONDARY'])
for key in config:
env[key] = config[key]
def on_failure(self, target, tries):
"""
:type target: TestTarget
:type tries: int
"""
if not tries and self.managed:
display.notice('If %s failed due to permissions, the test policy may need to be updated. '
'For help, consult @mattclay or @gundalow on GitHub or #ansible-devel on IRC.' % target.name)
@property
def inventory_hosts(self):
"""
:rtype: str | None
"""
return 'azure'
def get_config(config_path):
"""
:param config_path: str
:return: dict[str, str]
"""
with open(config_path, 'r') as config_fd:
lines = [line for line in config_fd.read().splitlines() if ':' in line and line.strip() and not line.strip().startswith('#')]
config = dict((kvp[0].strip(), kvp[1].strip()) for kvp in [line.split(':', 1) for line in lines])
rg_vars = (
'RESOURCE_GROUP',
'RESOURCE_GROUP_SECONDARY',
)
sp_vars = (
'AZURE_CLIENT_ID',
'AZURE_SECRET',
'AZURE_SUBSCRIPTION_ID',
'AZURE_TENANT',
)
ad_vars = (
'AZURE_AD_USER',
'AZURE_PASSWORD',
'AZURE_SUBSCRIPTION_ID',
)
rg_ok = all(var in config for var in rg_vars)
sp_ok = all(var in config for var in sp_vars)
ad_ok = all(var in config for var in ad_vars)
if not rg_ok:
raise ApplicationError('Resource groups must be defined with: %s' % ', '.join(sorted(rg_vars)))
if not sp_ok and not ad_ok:
raise ApplicationError('Credentials must be defined using either:\nService Principal: %s\nActive Directory: %s' % (
', '.join(sorted(sp_vars)), ', '.join(sorted(ad_vars))))
return config

View file

@ -56,6 +56,7 @@ class AnsibleCoreCI(object):
aws_platforms = (
'aws',
'azure',
'windows',
'freebsd',
'rhel',

View file

@ -15,10 +15,10 @@ except ImportError:
try:
# noinspection PyCompatibility
from urlparse import urlparse
from urlparse import urlparse, urlunparse, parse_qs
except ImportError:
# noinspection PyCompatibility, PyUnresolvedReferences
from urllib.parse import urlparse # pylint: disable=locally-disabled, ungrouped-imports
from urllib.parse import urlparse, urlunparse, parse_qs # pylint: disable=locally-disabled, ungrouped-imports
from lib.util import (
CommonConfig,