Tweak galaxy CI tests to bring stability (#72578)

This commit is contained in:
Jordan Borean 2020-11-11 13:46:57 +10:00 committed by GitHub
parent 44a38c9f33
commit a521ea0a13
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 82 additions and 99 deletions

View file

@ -56,12 +56,6 @@ options:
- The dependencies of the collection. - The dependencies of the collection.
type: dict type: dict
default: '{}' default: '{}'
wait:
description:
- Whether to wait for each collection's publish step to complete.
- When set to C(no), will only wait on the last publish task.
type: bool
default: false
author: author:
- Jordan Borean (@jborean93) - Jordan Borean (@jborean93)
''' '''
@ -89,6 +83,76 @@ import yaml
from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils._text import to_bytes from ansible.module_utils._text import to_bytes
from functools import partial
from multiprocessing import dummy as threading
def publish_collection(module, collection):
namespace = collection['namespace']
name = collection['name']
version = collection['version']
dependencies = collection['dependencies']
use_symlink = collection['use_symlink']
result = {}
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version))
b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
os.mkdir(b_collection_dir)
with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
fd.write(b"Collection readme")
galaxy_meta = {
'namespace': namespace,
'name': name,
'version': version,
'readme': 'README.md',
'authors': ['Collection author <name@email.com'],
'dependencies': dependencies,
'license': ['GPL-3.0-or-later'],
'repository': 'https://ansible.com/',
}
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
temp_fd.write(b"data")
if use_symlink:
os.mkdir(os.path.join(b_collection_dir, b'docs'))
os.mkdir(os.path.join(b_collection_dir, b'plugins'))
b_target_file = b'RE\xc3\x85DM\xc3\x88.md'
with open(os.path.join(b_collection_dir, b_target_file), mode='wb') as fd:
fd.write(b'data')
os.symlink(b_target_file, os.path.join(b_collection_dir, b_target_file + b'-link'))
os.symlink(temp_fd.name, os.path.join(b_collection_dir, b_target_file + b'-outside-link'))
os.symlink(os.path.join(b'..', b_target_file), os.path.join(b_collection_dir, b'docs', b_target_file))
os.symlink(os.path.join(b_collection_dir, b_target_file),
os.path.join(b_collection_dir, b'plugins', b_target_file))
os.symlink(b'docs', os.path.join(b_collection_dir, b'docs-link'))
release_filename = '%s-%s-%s.tar.gz' % (namespace, name, version)
collection_path = os.path.join(collection_dir, release_filename)
rc, stdout, stderr = module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir)
result['build'] = {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server', module.params['server']]
if module.params['token']:
publish_args.extend(['--token', module.params['token']])
rc, stdout, stderr = module.run_command(publish_args)
result['publish'] = {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
return result
def run_module(): def run_module():
@ -107,7 +171,6 @@ def run_module():
use_symlink=dict(type='bool', default=False), use_symlink=dict(type='bool', default=False),
), ),
), ),
wait=dict(type='bool', default=False),
) )
module = AnsibleModule( module = AnsibleModule(
@ -117,69 +180,9 @@ def run_module():
result = dict(changed=True, results=[]) result = dict(changed=True, results=[])
for idx, collection in enumerate(module.params['collections']): pool = threading.Pool(4)
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (collection['namespace'], collection['name'], publish_func = partial(publish_collection, module)
collection['version'])) result['results'] = pool.map(publish_func, module.params['collections'])
b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
os.mkdir(b_collection_dir)
with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
fd.write(b"Collection readme")
galaxy_meta = {
'namespace': collection['namespace'],
'name': collection['name'],
'version': collection['version'],
'readme': 'README.md',
'authors': ['Collection author <name@email.com'],
'dependencies': collection['dependencies'],
'license': ['GPL-3.0-or-later'],
'repository': 'https://ansible.com/',
}
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
temp_fd.write(b"data")
if collection['use_symlink']:
os.mkdir(os.path.join(b_collection_dir, b'docs'))
os.mkdir(os.path.join(b_collection_dir, b'plugins'))
b_target_file = b'RE\xc3\x85DM\xc3\x88.md'
with open(os.path.join(b_collection_dir, b_target_file), mode='wb') as fd:
fd.write(b'data')
os.symlink(b_target_file, os.path.join(b_collection_dir, b_target_file + b'-link'))
os.symlink(temp_fd.name, os.path.join(b_collection_dir, b_target_file + b'-outside-link'))
os.symlink(os.path.join(b'..', b_target_file), os.path.join(b_collection_dir, b'docs', b_target_file))
os.symlink(os.path.join(b_collection_dir, b_target_file),
os.path.join(b_collection_dir, b'plugins', b_target_file))
os.symlink(b'docs', os.path.join(b_collection_dir, b'docs-link'))
release_filename = '%s-%s-%s.tar.gz' % (collection['namespace'], collection['name'], collection['version'])
collection_path = os.path.join(collection_dir, release_filename)
rc, stdout, stderr = module.run_command(['ansible-galaxy', 'collection', 'build'], cwd=collection_dir)
result['results'].append({
'build': {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
})
# To save on time, skip the import wait until the last collection is being uploaded.
publish_args = ['ansible-galaxy', 'collection', 'publish', collection_path, '--server',
module.params['server']]
if module.params['token']:
publish_args.extend(['--token', module.params['token']])
if not module.params['wait'] and idx != (len(module.params['collections']) - 1):
publish_args.append('--no-wait')
rc, stdout, stderr = module.run_command(publish_args)
result['results'][-1]['publish'] = {
'rc': rc,
'stdout': stdout,
'stderr': stderr,
}
failed = bool(sum( failed = bool(sum(
r['build']['rc'] + r['publish']['rc'] for r in result['results'] r['build']['rc'] + r['publish']['rc'] for r in result['results']

View file

@ -316,7 +316,6 @@
- namespace: cache - namespace: cache
name: cache name: cache
version: 1.0.{{ cache_version_build }} version: 1.0.{{ cache_version_build }}
wait: yes
- name: make sure the cache version list is ignored on a collection version change - name: make sure the cache version list is ignored on a collection version change
command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv

View file

@ -71,37 +71,16 @@
v3: true v3: true
# We use a module for this so we can speed up the test time. # We use a module for this so we can speed up the test time.
- name: setup test collections for install and download test
loop:
# For pulp interactions, we only upload to galaxy_ng which shares # For pulp interactions, we only upload to galaxy_ng which shares
# the same repo and distribution with pulp_ansible # the same repo and distribution with pulp_ansible
# However, we use galaxy_ng only, since collections are unique across # However, we use galaxy_ng only, since collections are unique across
# pulp repositories, and galaxy_ng maintains a 2nd list of published collections # pulp repositories, and galaxy_ng maintains a 2nd list of published collections
- galaxy_ng - name: setup test collections for install and download test
setup_collections:
server: galaxy_ng
collections: '{{ collection_list }}'
environment: environment:
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
async: 300
poll: 0
register: setup_collections
setup_collections:
server: '{{ item }}'
collections: '{{ collection_list }}'
- name: Wait for setup_collections
async_status:
jid: '{{ item.ansible_job_id }}'
mode: status
retries: 300
delay: 1
loop: '{{ setup_collections.results }}'
register: setup_collections_wait
until: setup_collections_wait is finished
# The above setup_collections uses --no-wait
# pause for good measure.
- name: precautionary wait
pause:
seconds: 5
# Stores the cached test version number index as we run install many times # Stores the cached test version number index as we run install many times
- set_fact: - set_fact:

View file

@ -1,4 +1,6 @@
[galaxy] [galaxy]
# Ensures subsequent unstable reruns don't use the cached information causing another failure
cache_dir={{ remote_tmp_dir }}/galaxy_cache
server_list=pulp_v2,pulp_v3,galaxy_ng server_list=pulp_v2,pulp_v3,galaxy_ng
[galaxy_server.pulp_v2] [galaxy_server.pulp_v2]