Test ansible-galaxy against pulp/pulp_ansible (#69605)

* Test galaxy cli against pulp

* linting fix

* Renames and small fixes

* Better handling for resetting pulp

* Clean up some things, add a comment

* I can't spell

* Bump fallaxy, use alternate pulp image

* Only reset pulp when we're are executing against pulp

* Update for updated pulp container

* Update some comments with correct URLs and typos

* Linting fix

* Pin pulp-fedora31 to a digest

* Address review comments for documentation
This commit is contained in:
Matt Martz 2020-06-17 16:06:07 -05:00 committed by GitHub
parent cc071cdea6
commit fa1fb2d13b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 620 additions and 208 deletions

View file

@ -426,7 +426,7 @@ class GalaxyCLI(CLI):
'required': required, 'required': required,
} }
server_def = [('url', True), ('username', False), ('password', False), ('token', False), server_def = [('url', True), ('username', False), ('password', False), ('token', False),
('auth_url', False)] ('auth_url', False), ('v3', False)]
validate_certs = not context.CLIARGS['ignore_certs'] validate_certs = not context.CLIARGS['ignore_certs']
@ -447,6 +447,14 @@ class GalaxyCLI(CLI):
auth_url = server_options.pop('auth_url', None) auth_url = server_options.pop('auth_url', None)
token_val = server_options['token'] or NoTokenSentinel token_val = server_options['token'] or NoTokenSentinel
username = server_options['username'] username = server_options['username']
available_api_versions = None
v3 = server_options.pop('v3', None)
if v3:
# This allows a user to explicitly indicate the server uses the /v3 API
# This was added for testing against pulp_ansible and I'm not sure it has
# a practical purpose outside of this use case. As such, this option is not
# documented as of now
server_options['available_api_versions'] = {'v3': '/v3'}
# default case if no auth info is provided. # default case if no auth info is provided.
server_options['token'] = None server_options['token'] = None

View file

@ -168,7 +168,8 @@ class CollectionVersionMetadata:
class GalaxyAPI: class GalaxyAPI:
""" This class is meant to be used as a API client for an Ansible Galaxy server """ """ This class is meant to be used as a API client for an Ansible Galaxy server """
def __init__(self, galaxy, name, url, username=None, password=None, token=None, validate_certs=True): def __init__(self, galaxy, name, url, username=None, password=None, token=None, validate_certs=True,
available_api_versions=None):
self.galaxy = galaxy self.galaxy = galaxy
self.name = name self.name = name
self.username = username self.username = username
@ -176,7 +177,7 @@ class GalaxyAPI:
self.token = token self.token = token
self.api_server = url self.api_server = url
self.validate_certs = validate_certs self.validate_certs = validate_certs
self._available_api_versions = {} self._available_api_versions = available_api_versions or {}
display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs)) display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs))
@ -478,8 +479,16 @@ class GalaxyAPI:
wait = 2 wait = 2
while timeout == 0 or (time.time() - start) < timeout: while timeout == 0 or (time.time() - start) < timeout:
try:
data = self._call_galaxy(full_url, method='GET', auth_required=True, data = self._call_galaxy(full_url, method='GET', auth_required=True,
error_context_msg='Error when getting import task results at %s' % full_url) error_context_msg='Error when getting import task results at %s' % full_url)
except GalaxyError as e:
if e.http_code != 404:
raise
# The import job may not have started, and as such, the task url may not yet exist
display.vvv('Galaxy import process has not started, wait %s seconds before trying again' % wait)
time.sleep(wait)
continue
state = data.get('state', 'waiting') state = data.get('state', 'waiting')
@ -545,12 +554,10 @@ class GalaxyAPI:
relative_link = False relative_link = False
if 'v3' in self.available_api_versions: if 'v3' in self.available_api_versions:
api_path = self.available_api_versions['v3'] api_path = self.available_api_versions['v3']
results_key = 'data'
pagination_path = ['links', 'next'] pagination_path = ['links', 'next']
relative_link = True # AH pagination results are relative an not an absolute URI. relative_link = True # AH pagination results are relative an not an absolute URI.
else: else:
api_path = self.available_api_versions['v2'] api_path = self.available_api_versions['v2']
results_key = 'results'
pagination_path = ['next'] pagination_path = ['next']
n_url = _urljoin(self.api_server, api_path, 'collections', namespace, name, 'versions', '/') n_url = _urljoin(self.api_server, api_path, 'collections', namespace, name, 'versions', '/')
@ -559,6 +566,14 @@ class GalaxyAPI:
% (namespace, name, self.name, self.api_server) % (namespace, name, self.name, self.api_server)
data = self._call_galaxy(n_url, error_context_msg=error_context_msg) data = self._call_galaxy(n_url, error_context_msg=error_context_msg)
if 'data' in data:
# v3 automation-hub is the only known API that uses `data`
# since v3 pulp_ansible does not, we cannot rely on version
# to indicate which key to use
results_key = 'data'
else:
results_key = 'results'
versions = [] versions = []
while True: while True:
versions += [v['version'] for v in data[results_key]] versions += [v['version'] for v in data[results_key]]

View file

@ -102,8 +102,8 @@ matrix:
- env: T=linux/ubuntu1604/5 - env: T=linux/ubuntu1604/5
- env: T=linux/ubuntu1804/5 - env: T=linux/ubuntu1804/5
- env: T=fallaxy/2.7/1 - env: T=galaxy/2.7/1
- env: T=fallaxy/3.6/1 - env: T=galaxy/3.6/1
- env: T=generic/2.7/1 - env: T=generic/2.7/1
- env: T=generic/3.6/1 - env: T=generic/3.6/1

View file

@ -1,3 +1,3 @@
shippable/fallaxy/group1 shippable/galaxy/group1
shippable/fallaxy/smoketest shippable/galaxy/smoketest
cloud/fallaxy cloud/galaxy

View file

@ -88,7 +88,7 @@ from ansible.module_utils._text import to_bytes
def run_module(): def run_module():
module_args = dict( module_args = dict(
server=dict(type='str', required=True), server=dict(type='str', required=True),
token=dict(type='str', required=True), token=dict(type='str'),
collections=dict( collections=dict(
type='list', type='list',
elements='dict', elements='dict',
@ -108,7 +108,7 @@ def run_module():
supports_check_mode=False supports_check_mode=False
) )
result = dict(changed=True) result = dict(changed=True, results=[])
for idx, collection in enumerate(module.params['collections']): for idx, collection in enumerate(module.params['collections']):
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (collection['namespace'], collection['name'], collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (collection['namespace'], collection['name'],
@ -126,6 +126,7 @@ def run_module():
'readme': 'README.md', 'readme': 'README.md',
'authors': ['Collection author <name@email.com'], 'authors': ['Collection author <name@email.com'],
'dependencies': collection['dependencies'], 'dependencies': collection['dependencies'],
'license': ['GPL-3.0-or-later'],
} }
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd: with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict')) fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
@ -149,16 +150,34 @@ def run_module():
release_filename = '%s-%s-%s.tar.gz' % (collection['namespace'], collection['name'], collection['version']) release_filename = '%s-%s-%s.tar.gz' % (collection['namespace'], collection['name'], collection['version'])
collection_path = os.path.join(collection_dir, release_filename) collection_path = os.path.join(collection_dir, release_filename)
module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir) rc, stdout, stderr = module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir)
result['results'].append({
'build': {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
})
# To save on time, skip the import wait until the last collection is being uploaded. # To save on time, skip the import wait until the last collection is being uploaded.
publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server', publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server',
module.params['server'], '--token', module.params['token']] module.params['server']]
if module.params['token']:
publish_args.extend(['--token', module.params['token']])
if idx != (len(module.params['collections']) - 1): if idx != (len(module.params['collections']) - 1):
publish_args.append('--no-wait') publish_args.append('--no-wait')
module.run_command(publish_args) rc, stdout, stderr = module.run_command(publish_args)
result['results'][-1]['publish'] = {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
module.exit_json(**result) failed = bool(sum(
r['build']['rc'] + r['publish']['rc'] for r in result['results']
))
module.exit_json(failed=failed, **result)
def main(): def main():

View file

@ -131,9 +131,14 @@
register: fail_dep_mismatch register: fail_dep_mismatch
failed_when: '"Cannot meet dependency requirement ''fail_dep2.name:<0.0.5'' for collection fail_namespace.fail_collection" not in fail_dep_mismatch.stderr' failed_when: '"Cannot meet dependency requirement ''fail_dep2.name:<0.0.5'' for collection fail_namespace.fail_collection" not in fail_dep_mismatch.stderr'
- name: Find artifact url
uri:
url: '{{ test_server }}{{ vX }}collections/namespace3/name/versions/1.0.0/'
register: artifact_url_response
- name: download a collection for an offline install - {{ test_name }} - name: download a collection for an offline install - {{ test_name }}
get_url: get_url:
url: '{{ test_server }}custom/collections/namespace3-name-1.0.0.tar.gz' url: '{{ artifact_url_response.json.download_url }}'
dest: '{{ galaxy_dir }}/namespace3.tar.gz' dest: '{{ galaxy_dir }}/namespace3.tar.gz'
- name: install a collection from a tarball - {{ test_name }} - name: install a collection from a tarball - {{ test_name }}
@ -173,8 +178,13 @@
that: that:
- not fail_bad_tar_actual.stat.exists - not fail_bad_tar_actual.stat.exists
- name: Find artifact url
uri:
url: '{{ test_server }}{{ vX }}collections/namespace4/name/versions/1.0.0/'
register: artifact_url_response
- name: install a collection from a URI - {{ test_name }} - name: install a collection from a URI - {{ test_name }}
command: ansible-galaxy collection install '{{ test_server }}custom/collections/namespace4-name-1.0.0.tar.gz' {{ galaxy_verbosity }} command: ansible-galaxy collection install {{ artifact_url_response.json.download_url}} {{ galaxy_verbosity }}
register: install_uri register: install_uri
environment: environment:
ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'

View file

@ -19,22 +19,44 @@
src: ansible.cfg.j2 src: ansible.cfg.j2
dest: '{{ galaxy_dir }}/ansible.cfg' dest: '{{ galaxy_dir }}/ansible.cfg'
- name: configure pulp
include_tasks: pulp.yml
- name: run ansible-galaxy collection publish tests for {{ test_name }} - name: run ansible-galaxy collection publish tests for {{ test_name }}
include_tasks: publish.yml include_tasks: publish.yml
args:
apply:
environment:
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
vars: vars:
test_name: '{{ item.name }}' test_name: '{{ item.name }}'
test_server: '{{ item.server }}' test_server: '{{ item.server }}'
is_pulp: '{{ item.pulp|default(false) }}'
with_items: with_items:
- name: galaxy - name: galaxy
server: '{{ fallaxy_galaxy_server }}' server: '{{ fallaxy_galaxy_server }}'
- name: automation_hub - name: automation_hub
server: '{{ fallaxy_ah_server }}' server: '{{ fallaxy_ah_server }}'
- name: pulp_v2
server: '{{ pulp_v2_server }}'
pulp: true
- name: pulp_v3
server: '{{ pulp_v3_server }}'
pulp: true
# We use a module for this so we can speed up the test time. # We use a module for this so we can speed up the test time.
- name: setup test collections for install and download test - name: setup test collections for install and download test
loop:
- '{{ fallaxy_galaxy_server }}'
- pulp_v2
environment:
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
async: 300
poll: 0
register: setup_collections
setup_collections: setup_collections:
server: '{{ fallaxy_galaxy_server }}' server: '{{ item }}'
token: '{{ fallaxy_token }}' token: '{{ omit if item == "pulp_v2" else fallaxy_token }}'
collections: collections:
# Scenario to test out pre-release being ignored unless explicitly set and version pagination. # Scenario to test out pre-release being ignored unless explicitly set and version pagination.
- namespace: namespace1 - namespace: namespace1
@ -143,16 +165,43 @@
name: symlink name: symlink
use_symlink: yes use_symlink: yes
- name: Wait for setup_collections
async_status:
jid: '{{ item.ansible_job_id }}'
mode: status
retries: 300
delay: 1
loop: '{{ setup_collections.results }}'
register: setup_collections_wait
until: setup_collections_wait is finished
# The above setup_collections uses --no-wait
# pause for good measure.
- name: precautionary wait
pause:
seconds: 5
- name: run ansible-galaxy collection install tests for {{ test_name }} - name: run ansible-galaxy collection install tests for {{ test_name }}
include_tasks: install.yml include_tasks: install.yml
vars: vars:
test_name: '{{ item.name }}' test_name: '{{ item.name }}'
test_server: '{{ item.server }}' test_server: '{{ item.server }}'
vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}'
with_items: with_items:
- name: galaxy - name: galaxy
server: '{{ fallaxy_galaxy_server }}' server: '{{ fallaxy_galaxy_server }}'
- name: automation_hub - name: automation_hub
server: '{{ fallaxy_ah_server }}' server: '{{ fallaxy_ah_server }}'
v3: true
- name: pulp_v2
server: '{{ pulp_v2_server }}'
- name: pulp_v3
server: '{{ pulp_v3_server }}'
v3: true
- name: run ansible-galaxy collection download tests - name: run ansible-galaxy collection download tests
include_tasks: download.yml include_tasks: download.yml
args:
apply:
environment:
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'

View file

@ -5,6 +5,7 @@
chdir: '{{ galaxy_dir }}' chdir: '{{ galaxy_dir }}'
register: fail_no_token register: fail_no_token
failed_when: '"HTTP Code: 401" not in fail_no_token.stderr' failed_when: '"HTTP Code: 401" not in fail_no_token.stderr'
when: not is_pulp
- name: fail to publish with invalid token - {{ test_name }} - name: fail to publish with invalid token - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token fail {{ galaxy_verbosity }} command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token fail {{ galaxy_verbosity }}
@ -12,12 +13,24 @@
chdir: '{{ galaxy_dir }}' chdir: '{{ galaxy_dir }}'
register: fail_invalid_token register: fail_invalid_token
failed_when: '"HTTP Code: 401" not in fail_invalid_token.stderr' failed_when: '"HTTP Code: 401" not in fail_invalid_token.stderr'
when: not is_pulp
- name: publish collection - {{ test_name }} - name: publish collection - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token {{ fallaxy_token }} {{ galaxy_verbosity }} command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token {{ fallaxy_token }} {{ galaxy_verbosity }}
args: args:
chdir: '{{ galaxy_dir }}' chdir: '{{ galaxy_dir }}'
register: publish_collection register: fallaxy_publish_collection
when: not is_pulp
- name: publish collection - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: pulp_publish_collection
when: is_pulp
- set_fact:
publish_collection: '{{ pulp_publish_collection if pulp_publish_collection is not skipped else fallaxy_publish_collection }}'
- name: get result of publish collection - {{ test_name }} - name: get result of publish collection - {{ test_name }}
uri: uri:
@ -29,9 +42,9 @@
assert: assert:
that: that:
- '"Collection has been successfully published and imported to the Galaxy server" in publish_collection.stdout' - '"Collection has been successfully published and imported to the Galaxy server" in publish_collection.stdout'
- publish_collection_actual.json.metadata.name == 'my_collection' - publish_collection_actual.json.collection.name == 'my_collection'
- publish_collection_actual.json.metadata.namespace == 'ansible_test' - publish_collection_actual.json.namespace.name == 'ansible_test'
- publish_collection_actual.json.metadata.version == '1.0.0' - publish_collection_actual.json.version == '1.0.0'
- name: fail to publish existing collection version - {{ test_name }} - name: fail to publish existing collection version - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token {{ fallaxy_token }} {{ galaxy_verbosity }} command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token {{ fallaxy_token }} {{ galaxy_verbosity }}
@ -39,8 +52,22 @@
chdir: '{{ galaxy_dir }}' chdir: '{{ galaxy_dir }}'
register: fail_publish_existing register: fail_publish_existing
failed_when: '"Artifact already exists" not in fail_publish_existing.stderr' failed_when: '"Artifact already exists" not in fail_publish_existing.stderr'
when: not is_pulp
- name: fail to publish existing collection version - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fail_publish_existing
failed_when: fail_publish_existing is not failed
when: is_pulp
- name: reset published collections - {{ test_name }} - name: reset published collections - {{ test_name }}
uri: uri:
url: '{{ test_server }}custom/reset/' url: '{{ test_server }}custom/reset/'
method: POST method: POST
when: not is_pulp
- name: reset published collections - {{ test_name }}
include_tasks: pulp.yml
when: is_pulp

View file

@ -0,0 +1,154 @@
# These tasks configure pulp/pulp_ansible so that we can use the container
# This will also reset pulp between iterations
# The pulp container has a long start up time
# The first task to interact with pulp needs to wait until it responds appropriately
- name: list pulp distributions
uri:
url: '{{ pulp_api }}/pulp/api/v3/distributions/ansible/ansible/'
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distributions
until: pulp_distributions is successful
delay: 1
retries: 60
- when: pulp_distributions.json.count > 0
block:
- name: delete pulp distributions
uri:
url: '{{ pulp_api }}{{ distribution.pulp_href }}'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_delete_tasks
loop: '{{ pulp_distributions.json.results }}'
loop_control:
loop_var: distribution
label: '{{ distribution.name }}'
- name: wait for distribution delete
uri:
url: '{{ pulp_api }}{{ task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_delete
until: pulp_distribution_delete is successful and pulp_distribution_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
loop: '{{ pulp_distribution_delete_tasks.results }}'
loop_control:
loop_var: task
label: '{{ task.json.task }}'
- name: delete pulp repository
uri:
url: '{{ pulp_api }}{{ distribution.repository }}'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repository_delete_tasks
loop: '{{ pulp_distributions.json.results }}'
loop_control:
loop_var: distribution
label: '{{ distribution.name }}'
- name: wait for repository delete
uri:
url: '{{ pulp_api }}{{ task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repository_delete
until: pulp_repository_delete is successful and pulp_repository_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
loop: '{{ pulp_repository_delete_tasks.results }}'
loop_control:
loop_var: task
label: '{{ task.json.task }}'
- name: delete pulp orphans
uri:
url: '{{ pulp_api }}/pulp/api/v3/orphans/'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_orphans_delete_task
- name: wait for orphan delete
uri:
url: '{{ pulp_api }}{{ pulp_orphans_delete_task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_orphans_delete
until: pulp_orphans_delete is successful and pulp_orphans_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
- name: create pulp repo
uri:
url: '{{ pulp_api }}/pulp/api/v3/repositories/ansible/ansible/'
method: POST
body_format: json
body:
name: ansible_ci
status_code:
- 201
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repo
- name: create pulp distribution
uri:
url: '{{ pulp_api }}/pulp/api/v3/distributions/ansible/ansible/'
method: POST
body_format: json
body:
name: ansible_ci
base_path: ansible_ci
repository: '{{ pulp_api }}{{ pulp_repo.json.pulp_href }}'
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_task
- name: wait for distribution creation
uri:
url: '{{ pulp_api }}{{ pulp_distribution_task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
until: pulp_distribution is successful and pulp_distribution.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
register: pulp_distribution

View file

@ -1,5 +1,5 @@
[galaxy] [galaxy]
server_list=galaxy,automation_hub server_list=galaxy,automation_hub,pulp_v2,pulp_v3
[galaxy_server.galaxy] [galaxy_server.galaxy]
url={{ fallaxy_galaxy_server }} url={{ fallaxy_galaxy_server }}
@ -8,3 +8,14 @@ token={{ fallaxy_token }}
[galaxy_server.automation_hub] [galaxy_server.automation_hub]
url={{ fallaxy_ah_server }} url={{ fallaxy_ah_server }}
token={{ fallaxy_token }} token={{ fallaxy_token }}
[galaxy_server.pulp_v2]
url={{ pulp_v2_server }}
username={{ pulp_user }}
password={{ pulp_password }}
[galaxy_server.pulp_v3]
url={{ pulp_v3_server }}
v3=true
username={{ pulp_user }}
password={{ pulp_password }}

View file

@ -1,177 +0,0 @@
"""Fallaxy (ansible-galaxy) plugin for integration tests."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import uuid
from . import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from ..util import (
find_executable,
display,
)
from ..docker_util import (
docker_run,
docker_rm,
docker_inspect,
docker_pull,
get_docker_container_id,
)
class FallaxyProvider(CloudProvider):
"""Fallaxy plugin.
Sets up Fallaxy (ansible-galaxy) stub server for tests.
It's source source itself resides at: https://github.com/ansible/fallaxy-test-container
"""
DOCKER_SIMULATOR_NAME = 'fallaxy-stub'
def __init__(self, args):
"""
:type args: TestConfig
"""
super(FallaxyProvider, self).__init__(args)
if os.environ.get('ANSIBLE_FALLAXY_CONTAINER'):
self.image = os.environ.get('ANSIBLE_FALLAXY_CONTAINER')
else:
self.image = 'quay.io/ansible/fallaxy-test-container:2.0.1'
self.container_name = ''
def filter(self, targets, exclude):
"""Filter out the tests with the necessary config and res unavailable.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
docker_cmd = 'docker'
docker = find_executable(docker_cmd, required=False)
if docker:
return
skip = 'cloud/%s/' % self.platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which require the "%s" command: %s'
% (skip.rstrip('/'), docker_cmd, ', '.join(skipped)))
def setup(self):
"""Setup cloud resource before delegation and reg cleanup callback."""
super(FallaxyProvider, self).setup()
if self._use_static_config():
self._setup_static()
else:
self._setup_dynamic()
def get_docker_run_options(self):
"""Get additional options needed when delegating tests to a container.
:rtype: list[str]
"""
return ['--link', self.DOCKER_SIMULATOR_NAME] if self.managed else []
def cleanup(self):
"""Clean up the resource and temporary configs files after tests."""
if self.container_name:
docker_rm(self.args, self.container_name)
super(FallaxyProvider, self).cleanup()
def _setup_dynamic(self):
container_id = get_docker_container_id()
if container_id:
display.info('Running in docker container: %s' % container_id, verbosity=1)
self.container_name = self.DOCKER_SIMULATOR_NAME
results = docker_inspect(self.args, self.container_name)
if results and not results[0].get('State', {}).get('Running'):
docker_rm(self.args, self.container_name)
results = []
display.info('%s Fallaxy simulator docker container.'
% ('Using the existing' if results else 'Starting a new'),
verbosity=1)
fallaxy_port = 8080
fallaxy_token = str(uuid.uuid4()).replace('-', '')
if not results:
if self.args.docker or container_id:
publish_ports = []
else:
# publish the simulator ports when not running inside docker
publish_ports = [
'-p', ':'.join((str(fallaxy_port),) * 2),
]
if not os.environ.get('ANSIBLE_FALLAXY_CONTAINER'):
docker_pull(self.args, self.image)
docker_run(
self.args,
self.image,
['-d', '--name', self.container_name, '-e', 'FALLAXY_TOKEN=%s' % fallaxy_token] + publish_ports,
)
if self.args.docker:
fallaxy_host = self.DOCKER_SIMULATOR_NAME
elif container_id:
fallaxy_host = self._get_simulator_address()
display.info('Found Fallaxy simulator container address: %s' % fallaxy_host, verbosity=1)
else:
fallaxy_host = 'localhost'
self._set_cloud_config('FALLAXY_HOST', fallaxy_host)
self._set_cloud_config('FALLAXY_PORT', str(fallaxy_port))
self._set_cloud_config('FALLAXY_TOKEN', fallaxy_token)
def _get_simulator_address(self):
results = docker_inspect(self.args, self.container_name)
ipaddress = results[0]['NetworkSettings']['IPAddress']
return ipaddress
def _setup_static(self):
raise NotImplementedError()
class FallaxyEnvironment(CloudEnvironment):
"""Fallaxy environment plugin.
Updates integration test environment after delegation.
"""
def get_environment_config(self):
"""
:rtype: CloudEnvironmentConfig
"""
fallaxy_token = self._get_cloud_config('FALLAXY_TOKEN')
fallaxy_host = self._get_cloud_config('FALLAXY_HOST')
fallaxy_port = self._get_cloud_config('FALLAXY_PORT')
return CloudEnvironmentConfig(
ansible_vars=dict(
fallaxy_token=fallaxy_token,
fallaxy_galaxy_server='http://%s:%s/api/' % (fallaxy_host, fallaxy_port),
fallaxy_ah_server='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port),
),
env_vars=dict(
FALLAXY_TOKEN=fallaxy_token,
FALLAXY_GALAXY_SERVER='http://%s:%s/api/' % (fallaxy_host, fallaxy_port),
FALLAXY_AH_SERVER='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port),
),
)

View file

@ -0,0 +1,271 @@
"""Galaxy (ansible-galaxy) plugin for integration tests."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import tempfile
import uuid
from . import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from ..util import (
find_executable,
display,
)
from ..docker_util import (
docker_command,
docker_run,
docker_start,
docker_rm,
docker_inspect,
docker_pull,
get_docker_container_id,
)
SETTINGS = b'''
CONTENT_ORIGIN='pulp:80'
ANSIBLE_API_HOSTNAME='http://pulp:80'
ANSIBLE_CONTENT_HOSTNAME='http://pulp:80/pulp/content'
TOKEN_AUTH_DISABLED=True
'''
SET_ADMIN_PASSWORD = b'''#!/usr/bin/execlineb -S0
foreground {
redirfd -w 1 /dev/null
redirfd -w 2 /dev/null
export DJANGO_SETTINGS_MODULE pulpcore.app.settings
export PULP_CONTENT_ORIGIN localhost
s6-setuidgid postgres
/usr/local/bin/django-admin reset-admin-password --password password
}
'''
class GalaxyProvider(CloudProvider):
"""Galaxy plugin.
Sets up fallaxy and pulp (ansible-galaxy) servers for tests.
The fallaxy source itself resides at: https://github.com/ansible/fallaxy-test-container
The pulp source itself resides at: https://github.com/pulp/pulp-oci-images
"""
def __init__(self, args):
"""
:type args: TestConfig
"""
super(GalaxyProvider, self).__init__(args)
self.fallaxy = os.environ.get(
'ANSIBLE_FALLAXY_CONTAINER',
'quay.io/ansible/fallaxy-test-container:2.0.1'
)
self.pulp = os.environ.get(
'ANSIBLE_PULP_CONTAINER',
'docker.io/pulp/pulp-fedora31@sha256:71054f92fc9c986ba823d86b68631bafc84ae61b7832ce0be1f8e74423e56f64'
)
self.containers = []
def filter(self, targets, exclude):
"""Filter out the tests with the necessary config and res unavailable.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
docker_cmd = 'docker'
docker = find_executable(docker_cmd, required=False)
if docker:
return
skip = 'cloud/%s/' % self.platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which require the "%s" command: %s'
% (skip.rstrip('/'), docker_cmd, ', '.join(skipped)))
def setup(self):
"""Setup cloud resource before delegation and reg cleanup callback."""
super(GalaxyProvider, self).setup()
container_id = get_docker_container_id()
if container_id:
display.info('Running in docker container: %s' % container_id, verbosity=1)
f_results = docker_inspect(self.args, 'fallaxy-stub')
p_results = docker_inspect(self.args, 'pulp')
if f_results and not f_results[0].get('State', {}).get('Running'):
docker_rm(self.args, 'fallaxy-stub')
f_results = []
if p_results and not p_results[0].get('State', {}).get('Running'):
docker_rm(self.args, 'pulp')
p_results = []
display.info('%s fallaxy-stub docker container.'
% ('Using the existing' if f_results else 'Starting a new'),
verbosity=1)
fallaxy_port = 8080
fallaxy_token = str(uuid.uuid4()).replace('-', '')
if not f_results:
if self.args.docker or container_id:
publish_ports = []
else:
# publish the simulator ports when not running inside docker
publish_ports = [
'-p', ':'.join((str(fallaxy_port),) * 2),
]
docker_pull(self.args, self.fallaxy)
docker_run(
self.args,
self.fallaxy,
['-d', '--name', 'fallaxy-stub', '-e', 'FALLAXY_TOKEN=%s' % fallaxy_token] + publish_ports,
)
self.containers.append('fallaxy-stub')
if self.args.docker:
fallaxy_host = 'fallaxy-stub'
elif container_id:
fallaxy_host = self._get_simulator_address('fallaxy-stub')
display.info('Found Galaxy simulator container address: %s' % fallaxy_host, verbosity=1)
else:
fallaxy_host = 'localhost'
self._set_cloud_config('FALLAXY_HOST', fallaxy_host)
self._set_cloud_config('FALLAXY_PORT', str(fallaxy_port))
self._set_cloud_config('FALLAXY_TOKEN', fallaxy_token)
display.info('%s pulp docker container.'
% ('Using the existing' if p_results else 'Starting a new'),
verbosity=1)
pulp_port = 80
if not p_results:
if self.args.docker or container_id:
publish_ports = []
else:
# publish the simulator ports when not running inside docker
publish_ports = [
'-p', ':'.join((str(fallaxy_port),) * 2),
]
docker_pull(self.args, self.pulp)
# Create the container, don't run it, we need to inject configs before it starts
stdout, _dummy = docker_run(
self.args,
self.pulp,
['--name', 'pulp'] + publish_ports,
create_only=True
)
pulp_id = stdout.strip()
try:
# Inject our settings.py file
with tempfile.NamedTemporaryFile(delete=False) as settings:
settings.write(SETTINGS)
docker_command(self.args, ['cp', settings.name, '%s:/etc/pulp/settings.py' % pulp_id])
finally:
os.unlink(settings.name)
try:
# Inject our settings.py file
with tempfile.NamedTemporaryFile(delete=False) as admin_pass:
admin_pass.write(SET_ADMIN_PASSWORD)
docker_command(self.args, ['cp', admin_pass.name, '%s:/etc/cont-init.d/111-postgres' % pulp_id])
finally:
os.unlink(admin_pass.name)
# Start the container
docker_start(self.args, 'pulp', [])
self.containers.append('pulp')
if self.args.docker:
pulp_host = 'pulp'
elif container_id:
pulp_host = self._get_simulator_address('pulp')
display.info('Found Galaxy simulator container address: %s' % pulp_host, verbosity=1)
else:
pulp_host = 'localhost'
self._set_cloud_config('PULP_HOST', pulp_host)
self._set_cloud_config('PULP_PORT', str(pulp_port))
self._set_cloud_config('PULP_USER', 'admin')
self._set_cloud_config('PULP_PASSWORD', 'password')
def get_docker_run_options(self):
"""Get additional options needed when delegating tests to a container.
:rtype: list[str]
"""
return ['--link', 'fallaxy-stub', '--link', 'pulp'] # if self.managed else []
def cleanup(self):
"""Clean up the resource and temporary configs files after tests."""
for container_name in self.containers:
docker_rm(self.args, container_name)
super(GalaxyProvider, self).cleanup()
def _get_simulator_address(self, container_name):
results = docker_inspect(self.args, container_name)
ipaddress = results[0]['NetworkSettings']['IPAddress']
return ipaddress
class GalaxyEnvironment(CloudEnvironment):
"""Galaxy environment plugin.
Updates integration test environment after delegation.
"""
def get_environment_config(self):
"""
:rtype: CloudEnvironmentConfig
"""
fallaxy_token = self._get_cloud_config('FALLAXY_TOKEN')
fallaxy_host = self._get_cloud_config('FALLAXY_HOST')
fallaxy_port = self._get_cloud_config('FALLAXY_PORT')
pulp_user = self._get_cloud_config('PULP_USER')
pulp_password = self._get_cloud_config('PULP_PASSWORD')
pulp_host = self._get_cloud_config('PULP_HOST')
pulp_port = self._get_cloud_config('PULP_PORT')
return CloudEnvironmentConfig(
ansible_vars=dict(
fallaxy_token=fallaxy_token,
fallaxy_galaxy_server='http://%s:%s/api/' % (fallaxy_host, fallaxy_port),
fallaxy_ah_server='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port),
pulp_user=pulp_user,
pulp_password=pulp_password,
pulp_v2_server='http://%s:%s/pulp_ansible/galaxy/ansible_ci/api/' % (pulp_host, pulp_port),
pulp_v3_server='http://%s:%s/pulp_ansible/galaxy/ansible_ci/api/' % (pulp_host, pulp_port),
pulp_api='http://%s:%s' % (pulp_host, pulp_port),
),
env_vars=dict(
FALLAXY_TOKEN=fallaxy_token,
FALLAXY_GALAXY_SERVER='http://%s:%s/api/' % (fallaxy_host, fallaxy_port),
FALLAXY_AH_SERVER='http://%s:%s/api/automation-hub/' % (fallaxy_host, fallaxy_port),
PULP_USER=pulp_user,
PULP_PASSWORD=pulp_password,
PULP_V2_SERVER='http://%s:%s/pulp_ansible/galaxy/ansible_ci/api/' % (pulp_host, pulp_port),
PULP_V3_SERVER='http://%s:%s/pulp_ansible/galaxy/ansible_ci/api/' % (pulp_host, pulp_port),
),
)

View file

@ -135,12 +135,13 @@ def docker_get(args, container_id, src, dst):
options=['-i'], stdout=dst_fd, capture=True) options=['-i'], stdout=dst_fd, capture=True)
def docker_run(args, image, options, cmd=None): def docker_run(args, image, options, cmd=None, create_only=False):
""" """
:type args: EnvironmentConfig :type args: EnvironmentConfig
:type image: str :type image: str
:type options: list[str] | None :type options: list[str] | None
:type cmd: list[str] | None :type cmd: list[str] | None
:type create_only[bool] | False
:rtype: str | None, str | None :rtype: str | None, str | None
""" """
if not options: if not options:
@ -149,9 +150,14 @@ def docker_run(args, image, options, cmd=None):
if not cmd: if not cmd:
cmd = [] cmd = []
if create_only:
command = 'create'
else:
command = 'run'
for _iteration in range(1, 3): for _iteration in range(1, 3):
try: try:
return docker_command(args, ['run'] + options + [image] + cmd, capture=True) return docker_command(args, [command] + options + [image] + cmd, capture=True)
except SubprocessError as ex: except SubprocessError as ex:
display.error(ex) display.error(ex)
display.warning('Failed to run docker image "%s". Waiting a few seconds before trying again.' % image) display.warning('Failed to run docker image "%s". Waiting a few seconds before trying again.' % image)
@ -160,6 +166,24 @@ def docker_run(args, image, options, cmd=None):
raise ApplicationError('Failed to run docker image "%s".' % image) raise ApplicationError('Failed to run docker image "%s".' % image)
def docker_start(args, container_id, options): # type: (EnvironmentConfig, str, t.List[str]) -> (t.Optional[str], t.Optional[str])
"""
Start a docker container by name or ID
"""
if not options:
options = []
for _iteration in range(1, 3):
try:
return docker_command(args, ['start'] + options + [container_id], capture=True)
except SubprocessError as ex:
display.error(ex)
display.warning('Failed to start docker container "%s". Waiting a few seconds before trying again.' % container_id)
time.sleep(3)
raise ApplicationError('Failed to run docker container "%s".' % container_id)
def docker_images(args, image): def docker_images(args, image):
""" """
:param args: CommonConfig :param args: CommonConfig
@ -275,7 +299,7 @@ def docker_version(args):
return json.loads(stdout) return json.loads(stdout)
def docker_command(args, cmd, capture=False, stdin=None, stdout=None, always=False): def docker_command(args, cmd, capture=False, stdin=None, stdout=None, always=False, data=None):
""" """
:type args: CommonConfig :type args: CommonConfig
:type cmd: list[str] :type cmd: list[str]
@ -283,10 +307,11 @@ def docker_command(args, cmd, capture=False, stdin=None, stdout=None, always=Fal
:type stdin: file | None :type stdin: file | None
:type stdout: file | None :type stdout: file | None
:type always: bool :type always: bool
:type data: str | None
:rtype: str | None, str | None :rtype: str | None, str | None
""" """
env = docker_environment() env = docker_environment()
return run_command(args, ['docker'] + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, always=always) return run_command(args, ['docker'] + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, always=always, data=data)
def docker_environment(): def docker_environment():