Add Azure provider support to ansible-test and CI.

ci_complete

test/runner/lib/cloud/aws.py

@@ -79,7 +79,7 @@ class AwsCloudProvider(CloudProvider):
         """
         :rtype: AnsibleCoreCI
         """
-        return AnsibleCoreCI(self.args, 'aws', 'sts', persist=False, stage=self.args.remote_stage)
+        return AnsibleCoreCI(self.args, 'aws', 'sts', persist=False, stage=self.args.remote_stage, provider=self.args.remote_provider)
 
 
 class AwsCloudEnvironment(CloudEnvironment):

test/runner/lib/cloud/azure.py

@@ -114,7 +114,7 @@ class AzureCloudProvider(CloudProvider):
         """
         :rtype: AnsibleCoreCI
         """
-        return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage)
+        return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage, provider=self.args.remote_provider)
 
 
 class AzureCloudEnvironment(CloudEnvironment):

test/runner/lib/config.py

@@ -48,9 +48,13 @@ class EnvironmentConfig(CommonConfig):
         self.tox_sitepackages = args.tox_sitepackages  # type: bool
 
         self.remote_stage = args.remote_stage  # type: str
+        self.remote_provider = args.remote_provider  # type: str
         self.remote_aws_region = args.remote_aws_region  # type: str
         self.remote_terminate = args.remote_terminate  # type: str
 
+        if self.remote_provider == 'default':
+            self.remote_provider = None
+
         self.requirements = args.requirements  # type: bool
 
         if self.python == 'default':

test/runner/lib/core_ci.py

@@ -36,7 +36,7 @@ AWS_ENDPOINTS = {
 
 class AnsibleCoreCI(object):
     """Client for Ansible Core CI services."""
-    def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None):
+    def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None):
         """
         :type args: EnvironmentConfig
         :type platform: str
@@ -57,23 +57,42 @@ class AnsibleCoreCI(object):
         self.max_threshold = 1
         self.name = name if name else '%s-%s' % (self.platform, self.version)
         self.ci_key = os.path.expanduser('~/.ansible-core-ci.key')
+        self.resource = 'jobs'
 
-        aws_platforms = (
+        # Assign each supported platform to one provider.
-            'aws',
+        # This is used to determine the provider from the platform when no provider is specified.
-            'azure',
+        providers = dict(
-            'windows',
+            aws=(
-            'freebsd',
+                'aws',
-            'rhel',
+                'azure',
-            'vyos',
+                'windows',
-            'junos',
+                'freebsd',
-            'ios',
+                'rhel',
+                'vyos',
+                'junos',
+                'ios',
+            ),
+            azure=(
+            ),
+            parallels=(
+                'osx',
+            ),
         )
 
-        osx_platforms = (
+        if provider:
-            'osx',
+            # override default provider selection (not all combinations are valid)
-        )
+            self.provider = provider
+        else:
+            for candidate in providers:
+                if platform in providers[candidate]:
+                    # assign default provider based on platform
+                    self.provider = candidate
+                    break
+
+        if self.provider in ('aws', 'azure'):
+            if self.provider != 'aws':
+                self.resource = self.provider
 
-        if self.platform in aws_platforms:
             if args.remote_aws_region:
                 # permit command-line override of region selection
                 region = args.remote_aws_region
@@ -97,7 +116,7 @@ class AnsibleCoreCI(object):
             else:
                 self.ssh_key = SshKey(args)
                 self.port = 22
-        elif self.platform in osx_platforms:
+        elif self.provider == 'parallels':
             self.endpoints = self._get_parallels_endpoints()
             self.max_threshold = 6
 
@@ -106,7 +125,7 @@ class AnsibleCoreCI(object):
         else:
             raise ApplicationError('Unsupported platform: %s' % platform)
 
-        self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s' % (self.name, self.stage))
+        self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage))
 
         if persist and load and self._load():
             try:
@@ -292,7 +311,7 @@ class AnsibleCoreCI(object):
 
     @property
     def _uri(self):
-        return '%s/%s/jobs/%s' % (self.endpoint, self.stage, self.instance_id)
+        return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.resource, self.instance_id)
 
     def _start(self, auth):
         """Start instance."""

test/runner/lib/delegation.py

@@ -280,7 +280,7 @@ def delegate_remote(args, exclude, require):
     platform = parts[0]
     version = parts[1]
 
-    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)
+    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider)
     success = False
 
     try:

test/runner/lib/executor.py

@@ -406,7 +406,7 @@ def network_start(args, platform, version):
     :type version: str
     :rtype: AnsibleCoreCI
     """
-    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)
+    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider)
     core_ci.start()
 
     return core_ci.save()
@@ -420,7 +420,7 @@ def network_run(args, platform, version, config):
     :type config: dict[str, str]
     :rtype: AnsibleCoreCI
     """
-    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, load=False)
+    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider, load=False)
     core_ci.load(config)
     core_ci.wait()
 
@@ -548,7 +548,7 @@ def windows_start(args, version):
     :type version: str
     :rtype: AnsibleCoreCI
     """
-    core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage)
+    core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, provider=args.remote_provider)
     core_ci.start()
 
     return core_ci.save()
@@ -561,7 +561,7 @@ def windows_run(args, version, config):
     :type config: dict[str, str]
     :rtype: AnsibleCoreCI
     """
-    core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, load=False)
+    core_ci = AnsibleCoreCI(args, 'windows', version, stage=args.remote_stage, provider=args.remote_provider, load=False)
     core_ci.load(config)
     core_ci.wait()
 

test/runner/lib/manage_ci.py

@@ -131,7 +131,12 @@ class ManagePosixCI(object):
             self.ssh_args += ['-o', '%s=%s' % (ssh_option, ssh_options[ssh_option])]
 
         if self.core_ci.platform == 'freebsd':
-            self.become = ['su', '-l', 'root', '-c']
+            if self.core_ci.provider == 'aws':
+                self.become = ['su', '-l', 'root', '-c']
+            elif self.core_ci.provider == 'azure':
+                self.become = ['sudo', '-in', 'sh', '-c']
+            else:
+                raise NotImplementedError('provider %s has not been implemented' % self.core_ci.provider)
         elif self.core_ci.platform == 'osx':
             self.become = ['sudo', '-in', 'PATH=/usr/local/bin:$PATH']
         elif self.core_ci.platform == 'rhel':

test/runner/setup/remote.sh

@@ -10,6 +10,7 @@ cd ~/
 
 if [ "${platform}" = "freebsd" ]; then
     while true; do
+        env ASSUME_ALWAYS_YES=YES pkg bootstrap && \
         pkg install -y \
             bash \
             curl \

test/runner/test.py

@@ -514,6 +514,7 @@ def add_environments(parser, tox_version=False, tox_only=False):
             docker=None,
             remote=None,
             remote_stage=None,
+            remote_provider=None,
             remote_aws_region=None,
             remote_terminate=None,
         )
@@ -540,6 +541,12 @@ def add_environments(parser, tox_version=False, tox_only=False):
                         choices=['prod', 'dev'],
                         default='prod')
 
+    remote.add_argument('--remote-provider',
+                        metavar='PROVIDER',
+                        help='remote provider to use: %(choices)s',
+                        choices=['default', 'aws', 'azure', 'parallels'],
+                        default='default')
+
     remote.add_argument('--remote-aws-region',
                         metavar='REGION',
                         help='remote aws region to use: %(choices)s (default: auto)',

test/utils/shippable/freebsd.sh

@@ -14,7 +14,10 @@ else
     target="posix/ci/"
 fi
 
+stage="${S:-prod}"
+provider="${P:-default}"
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --exclude "posix/ci/cloud/" \
-    --remote "${platform}/${version}" --remote-terminate always
+    --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/network.sh

@@ -12,6 +12,9 @@ fi
 
 target="network/ci/"
 
+stage="${S:-prod}"
+provider="${P:-default}"
+
 # python versions to test in order
 # all versions run full tests
 python_versions=(
@@ -50,5 +53,6 @@ for version in "${python_versions[@]}"; do
 
     # shellcheck disable=SC2086
     ansible-test network-integration --color -v --retry-on-error "${target}" --docker default --python "${version}" \
-        ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" --remote-terminate "${terminate}"
+        ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" \
+        --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
 done

test/utils/shippable/osx.sh

@@ -14,7 +14,10 @@ else
     target="posix/ci/"
 fi
 
+stage="${S:-prod}"
+provider="${P:-default}"
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --exclude "posix/ci/cloud/" \
-    --remote "${platform}/${version}" --remote-terminate always
+    --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/rhel.sh

@@ -14,7 +14,10 @@ else
     target="posix/ci/"
 fi
 
+stage="${S:-prod}"
+provider="${P:-default}"
+
 # shellcheck disable=SC2086
 ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
     --exclude "posix/ci/cloud/" \
-    --remote "${platform}/${version}" --remote-terminate always
+    --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

test/utils/shippable/windows.sh

@@ -7,6 +7,9 @@ IFS='/:' read -ra args <<< "$1"
 
 target="windows/ci/group${args[1]}/"
 
+stage="${S:-prod}"
+provider="${P:-default}"
+
 # python versions to test in order
 # python 2.7 runs full tests while other versions run minimal tests
 python_versions=(
@@ -76,5 +79,6 @@ for version in "${python_versions[@]}"; do
 
     # shellcheck disable=SC2086
     ansible-test windows-integration --color -v --retry-on-error "${ci}" --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-        "${platforms[@]}" --changed-all-target "${changed_all_target}" --remote-terminate "${terminate}"
+        "${platforms[@]}" --changed-all-target "${changed_all_target}" \
+        --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
 done