Tweak galaxy CI tests to bring stability (#72578)
This commit is contained in:
parent
44a38c9f33
commit
a521ea0a13
4 changed files with 82 additions and 99 deletions
|
@ -56,12 +56,6 @@ options:
|
||||||
- The dependencies of the collection.
|
- The dependencies of the collection.
|
||||||
type: dict
|
type: dict
|
||||||
default: '{}'
|
default: '{}'
|
||||||
wait:
|
|
||||||
description:
|
|
||||||
- Whether to wait for each collection's publish step to complete.
|
|
||||||
- When set to C(no), will only wait on the last publish task.
|
|
||||||
type: bool
|
|
||||||
default: false
|
|
||||||
author:
|
author:
|
||||||
- Jordan Borean (@jborean93)
|
- Jordan Borean (@jborean93)
|
||||||
'''
|
'''
|
||||||
|
@ -89,6 +83,76 @@ import yaml
|
||||||
|
|
||||||
from ansible.module_utils.basic import AnsibleModule
|
from ansible.module_utils.basic import AnsibleModule
|
||||||
from ansible.module_utils._text import to_bytes
|
from ansible.module_utils._text import to_bytes
|
||||||
|
from functools import partial
|
||||||
|
from multiprocessing import dummy as threading
|
||||||
|
|
||||||
|
|
||||||
|
def publish_collection(module, collection):
|
||||||
|
namespace = collection['namespace']
|
||||||
|
name = collection['name']
|
||||||
|
version = collection['version']
|
||||||
|
dependencies = collection['dependencies']
|
||||||
|
use_symlink = collection['use_symlink']
|
||||||
|
|
||||||
|
result = {}
|
||||||
|
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version))
|
||||||
|
b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
|
||||||
|
os.mkdir(b_collection_dir)
|
||||||
|
|
||||||
|
with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
|
||||||
|
fd.write(b"Collection readme")
|
||||||
|
|
||||||
|
galaxy_meta = {
|
||||||
|
'namespace': namespace,
|
||||||
|
'name': name,
|
||||||
|
'version': version,
|
||||||
|
'readme': 'README.md',
|
||||||
|
'authors': ['Collection author <name@email.com'],
|
||||||
|
'dependencies': dependencies,
|
||||||
|
'license': ['GPL-3.0-or-later'],
|
||||||
|
'repository': 'https://ansible.com/',
|
||||||
|
}
|
||||||
|
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
|
||||||
|
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
|
||||||
|
temp_fd.write(b"data")
|
||||||
|
|
||||||
|
if use_symlink:
|
||||||
|
os.mkdir(os.path.join(b_collection_dir, b'docs'))
|
||||||
|
os.mkdir(os.path.join(b_collection_dir, b'plugins'))
|
||||||
|
b_target_file = b'RE\xc3\x85DM\xc3\x88.md'
|
||||||
|
with open(os.path.join(b_collection_dir, b_target_file), mode='wb') as fd:
|
||||||
|
fd.write(b'data')
|
||||||
|
|
||||||
|
os.symlink(b_target_file, os.path.join(b_collection_dir, b_target_file + b'-link'))
|
||||||
|
os.symlink(temp_fd.name, os.path.join(b_collection_dir, b_target_file + b'-outside-link'))
|
||||||
|
os.symlink(os.path.join(b'..', b_target_file), os.path.join(b_collection_dir, b'docs', b_target_file))
|
||||||
|
os.symlink(os.path.join(b_collection_dir, b_target_file),
|
||||||
|
os.path.join(b_collection_dir, b'plugins', b_target_file))
|
||||||
|
os.symlink(b'docs', os.path.join(b_collection_dir, b'docs-link'))
|
||||||
|
|
||||||
|
release_filename = '%s-%s-%s.tar.gz' % (namespace, name, version)
|
||||||
|
collection_path = os.path.join(collection_dir, release_filename)
|
||||||
|
rc, stdout, stderr = module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir)
|
||||||
|
result['build'] = {
|
||||||
|
'rc': rc,
|
||||||
|
'stdout': stdout,
|
||||||
|
'stderr': stderr,
|
||||||
|
}
|
||||||
|
|
||||||
|
publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server', module.params['server']]
|
||||||
|
if module.params['token']:
|
||||||
|
publish_args.extend(['--token', module.params['token']])
|
||||||
|
|
||||||
|
rc, stdout, stderr = module.run_command(publish_args)
|
||||||
|
result['publish'] = {
|
||||||
|
'rc': rc,
|
||||||
|
'stdout': stdout,
|
||||||
|
'stderr': stderr,
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def run_module():
|
def run_module():
|
||||||
|
@ -107,7 +171,6 @@ def run_module():
|
||||||
use_symlink=dict(type='bool', default=False),
|
use_symlink=dict(type='bool', default=False),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
wait=dict(type='bool', default=False),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
module = AnsibleModule(
|
module = AnsibleModule(
|
||||||
|
@ -117,69 +180,9 @@ def run_module():
|
||||||
|
|
||||||
result = dict(changed=True, results=[])
|
result = dict(changed=True, results=[])
|
||||||
|
|
||||||
for idx, collection in enumerate(module.params['collections']):
|
pool = threading.Pool(4)
|
||||||
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (collection['namespace'], collection['name'],
|
publish_func = partial(publish_collection, module)
|
||||||
collection['version']))
|
result['results'] = pool.map(publish_func, module.params['collections'])
|
||||||
b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
|
|
||||||
os.mkdir(b_collection_dir)
|
|
||||||
|
|
||||||
with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
|
|
||||||
fd.write(b"Collection readme")
|
|
||||||
|
|
||||||
galaxy_meta = {
|
|
||||||
'namespace': collection['namespace'],
|
|
||||||
'name': collection['name'],
|
|
||||||
'version': collection['version'],
|
|
||||||
'readme': 'README.md',
|
|
||||||
'authors': ['Collection author <name@email.com'],
|
|
||||||
'dependencies': collection['dependencies'],
|
|
||||||
'license': ['GPL-3.0-or-later'],
|
|
||||||
'repository': 'https://ansible.com/',
|
|
||||||
}
|
|
||||||
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
|
|
||||||
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
|
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
|
|
||||||
temp_fd.write(b"data")
|
|
||||||
|
|
||||||
if collection['use_symlink']:
|
|
||||||
os.mkdir(os.path.join(b_collection_dir, b'docs'))
|
|
||||||
os.mkdir(os.path.join(b_collection_dir, b'plugins'))
|
|
||||||
b_target_file = b'RE\xc3\x85DM\xc3\x88.md'
|
|
||||||
with open(os.path.join(b_collection_dir, b_target_file), mode='wb') as fd:
|
|
||||||
fd.write(b'data')
|
|
||||||
|
|
||||||
os.symlink(b_target_file, os.path.join(b_collection_dir, b_target_file + b'-link'))
|
|
||||||
os.symlink(temp_fd.name, os.path.join(b_collection_dir, b_target_file + b'-outside-link'))
|
|
||||||
os.symlink(os.path.join(b'..', b_target_file), os.path.join(b_collection_dir, b'docs', b_target_file))
|
|
||||||
os.symlink(os.path.join(b_collection_dir, b_target_file),
|
|
||||||
os.path.join(b_collection_dir, b'plugins', b_target_file))
|
|
||||||
os.symlink(b'docs', os.path.join(b_collection_dir, b'docs-link'))
|
|
||||||
|
|
||||||
release_filename = '%s-%s-%s.tar.gz' % (collection['namespace'], collection['name'], collection['version'])
|
|
||||||
collection_path = os.path.join(collection_dir, release_filename)
|
|
||||||
rc, stdout, stderr = module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir)
|
|
||||||
result['results'].append({
|
|
||||||
'build': {
|
|
||||||
'rc': rc,
|
|
||||||
'stdout': stdout,
|
|
||||||
'stderr': stderr,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
# To save on time, skip the import wait until the last collection is being uploaded.
|
|
||||||
publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server',
|
|
||||||
module.params['server']]
|
|
||||||
if module.params['token']:
|
|
||||||
publish_args.extend(['--token', module.params['token']])
|
|
||||||
if not module.params['wait'] and idx != (len(module.params['collections']) - 1):
|
|
||||||
publish_args.append('--no-wait')
|
|
||||||
rc, stdout, stderr = module.run_command(publish_args)
|
|
||||||
result['results'][-1]['publish'] = {
|
|
||||||
'rc': rc,
|
|
||||||
'stdout': stdout,
|
|
||||||
'stderr': stderr,
|
|
||||||
}
|
|
||||||
|
|
||||||
failed = bool(sum(
|
failed = bool(sum(
|
||||||
r['build']['rc'] + r['publish']['rc'] for r in result['results']
|
r['build']['rc'] + r['publish']['rc'] for r in result['results']
|
||||||
|
|
|
@ -316,7 +316,6 @@
|
||||||
- namespace: cache
|
- namespace: cache
|
||||||
name: cache
|
name: cache
|
||||||
version: 1.0.{{ cache_version_build }}
|
version: 1.0.{{ cache_version_build }}
|
||||||
wait: yes
|
|
||||||
|
|
||||||
- name: make sure the cache version list is ignored on a collection version change
|
- name: make sure the cache version list is ignored on a collection version change
|
||||||
command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv
|
command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv
|
||||||
|
|
|
@ -71,37 +71,16 @@
|
||||||
v3: true
|
v3: true
|
||||||
|
|
||||||
# We use a module for this so we can speed up the test time.
|
# We use a module for this so we can speed up the test time.
|
||||||
- name: setup test collections for install and download test
|
|
||||||
loop:
|
|
||||||
# For pulp interactions, we only upload to galaxy_ng which shares
|
# For pulp interactions, we only upload to galaxy_ng which shares
|
||||||
# the same repo and distribution with pulp_ansible
|
# the same repo and distribution with pulp_ansible
|
||||||
# However, we use galaxy_ng only, since collections are unique across
|
# However, we use galaxy_ng only, since collections are unique across
|
||||||
# pulp repositories, and galaxy_ng maintains a 2nd list of published collections
|
# pulp repositories, and galaxy_ng maintains a 2nd list of published collections
|
||||||
- galaxy_ng
|
- name: setup test collections for install and download test
|
||||||
|
setup_collections:
|
||||||
|
server: galaxy_ng
|
||||||
|
collections: '{{ collection_list }}'
|
||||||
environment:
|
environment:
|
||||||
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
|
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
|
||||||
async: 300
|
|
||||||
poll: 0
|
|
||||||
register: setup_collections
|
|
||||||
setup_collections:
|
|
||||||
server: '{{ item }}'
|
|
||||||
collections: '{{ collection_list }}'
|
|
||||||
|
|
||||||
- name: Wait for setup_collections
|
|
||||||
async_status:
|
|
||||||
jid: '{{ item.ansible_job_id }}'
|
|
||||||
mode: status
|
|
||||||
retries: 300
|
|
||||||
delay: 1
|
|
||||||
loop: '{{ setup_collections.results }}'
|
|
||||||
register: setup_collections_wait
|
|
||||||
until: setup_collections_wait is finished
|
|
||||||
|
|
||||||
# The above setup_collections uses --no-wait
|
|
||||||
# pause for good measure.
|
|
||||||
- name: precautionary wait
|
|
||||||
pause:
|
|
||||||
seconds: 5
|
|
||||||
|
|
||||||
# Stores the cached test version number index as we run install many times
|
# Stores the cached test version number index as we run install many times
|
||||||
- set_fact:
|
- set_fact:
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
[galaxy]
|
[galaxy]
|
||||||
|
# Ensures subsequent unstable reruns don't use the cached information causing another failure
|
||||||
|
cache_dir={{ remote_tmp_dir }}/galaxy_cache
|
||||||
server_list=pulp_v2,pulp_v3,galaxy_ng
|
server_list=pulp_v2,pulp_v3,galaxy_ng
|
||||||
|
|
||||||
[galaxy_server.pulp_v2]
|
[galaxy_server.pulp_v2]
|
||||||
|
|
Loading…
Reference in a new issue