Galaxy server update (#72286)

* Update galaxy server image used in CI

* Fix port and migration issue

* Add delete step, still need to deal with pagination

* Make cleanup more efficient

* Remove testing code
This commit is contained in:
Jordan Borean 2020-10-28 09:17:40 +10:00 committed by GitHub
parent d6115887fa
commit 4856ab0e68
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 290 additions and 267 deletions

View file

@ -0,0 +1,209 @@
#!/usr/bin/python
# Copyright: (c) 2020, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = '''
---
module: reset_pulp
short_description: Resets pulp back to the initial state
description:
- See short_description
options:
pulp_api:
description:
- The Pulp API endpoint.
required: yes
type: str
galaxy_ng_server:
description:
- The Galaxy NG API endpoint.
required: yes
type: str
url_username:
description:
- The username to use when authenticating against Pulp.
required: yes
type: str
url_password:
description:
- The password to use when authenticating against Pulp.
required: yes
type: str
repository:
description:
- The name of the repository to create.
- This should match C(GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH) in C(/etc/pulp/settings.py) or use the default of
C(published).
required: yes
type: str
namespaces:
description:
- A list of namespaces to create.
required: yes
type: list
elements: str
author:
- Jordan Borean (@jborean93)
'''
EXAMPLES = '''
- name: reset pulp content
reset_pulp:
pulp_api: http://galaxy:24817
galaxy_ng_server: http://galaxy/api/galaxy/
url_username: username
url_password: password
repository: published
namespaces:
- namespace1
- namespace2
'''
RETURN = '''
#
'''
import json
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.urls import fetch_url
from ansible.module_utils.common.text.converters import to_text
def invoke_api(module, url, method='GET', data=None, status_codes=None):
status_codes = status_codes or [200]
headers = {}
if data:
headers['Content-Type'] = 'application/json'
data = json.dumps(data)
resp, info = fetch_url(module, url, method=method, data=data, headers=headers)
if info['status'] not in status_codes:
module.fail_json(url=url, **info)
data = to_text(resp.read())
if data:
return json.loads(data)
def delete_galaxy_namespace(namespace, module):
""" Deletes the galaxy ng namespace specified. """
ns_uri = '%sv3/namespaces/%s/' % (module.params['galaxy_ng_server'], namespace)
invoke_api(module, ns_uri, method='DELETE', status_codes=[204])
def delete_pulp_distribution(distribution, module):
""" Deletes the pulp distribution at the URI specified. """
task_info = invoke_api(module, distribution, method='DELETE', status_codes=[202])
wait_pulp_task(task_info['task'], module)
def delete_pulp_orphans(module):
""" Deletes any orphaned pulp objects. """
orphan_uri = module.params['pulp_api'] + '/pulp/api/v3/orphans/'
task_info = invoke_api(module, orphan_uri, method='DELETE', status_codes=[202])
wait_pulp_task(task_info['task'], module)
def delete_pulp_repository(repository, module):
""" Deletes the pulp repository at the URI specified. """
task_info = invoke_api(module, repository, method='DELETE', status_codes=[202])
wait_pulp_task(task_info['task'], module)
def get_galaxy_namespaces(module):
""" Gets a list of galaxy namespaces. """
# No pagination has been implemented, shouldn't need unless we ever exceed 100 namespaces.
namespace_uri = module.params['galaxy_ng_server'] + 'v3/namespaces/?limit=100&offset=0'
ns_info = invoke_api(module, namespace_uri)
return [n['name'] for n in ns_info['data']]
def get_pulp_distributions(module):
""" Gets a list of all the pulp distributions. """
distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/'
distro_info = invoke_api(module, distro_uri)
return [module.params['pulp_api'] + r['pulp_href'] for r in distro_info['results']]
def get_pulp_repositories(module):
""" Gets a list of all the pulp repositories. """
repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/'
repo_info = invoke_api(module, repo_uri)
return [module.params['pulp_api'] + r['pulp_href'] for r in repo_info['results']]
def new_galaxy_namespace(name, module):
""" Creates a new namespace in Galaxy NG. """
ns_uri = module.params['galaxy_ng_server'] + 'v3/_ui/namespaces/'
data = {'name': name, 'groups': [{'name': 'system:partner-engineers', 'object_permissions':
['add_namespace', 'change_namespace', 'upload_to_namespace']}]}
ns_info = invoke_api(module, ns_uri, method='POST', data=data, status_codes=[201])
return ns_info['id']
def new_pulp_repository(name, module):
""" Creates a new pulp repository. """
repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/'
data = {'name': name}
repo_info = invoke_api(module, repo_uri, method='POST', data=data, status_codes=[201])
return module.params['pulp_api'] + repo_info['pulp_href']
def new_pulp_distribution(name, base_path, repository, module):
""" Creates a new pulp distribution for a repository. """
distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/'
data = {'name': name, 'base_path': base_path, 'repository': repository}
task_info = invoke_api(module, distro_uri, method='POST', data=data, status_codes=[202])
task_info = wait_pulp_task(task_info['task'], module)
return module.params['pulp_api'] + task_info['created_resources'][0]
def wait_pulp_task(task, module):
""" Waits for a pulp import task to finish. """
while True:
task_info = invoke_api(module, module.params['pulp_api'] + task)
if task_info['finished_at'] is not None:
break
return task_info
def main():
module_args = dict(
pulp_api=dict(type='str', required=True),
galaxy_ng_server=dict(type='str', required=True),
url_username=dict(type='str', required=True),
url_password=dict(type='str', required=True, no_log=True),
repository=dict(type='str', required=True),
namespaces=dict(type='list', elements='str', required=True),
)
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=False
)
module.params['force_basic_auth'] = True
[delete_pulp_distribution(d, module) for d in get_pulp_distributions(module)]
[delete_pulp_repository(r, module) for r in get_pulp_repositories(module)]
delete_pulp_orphans(module)
[delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)]
repo_href = new_pulp_repository(module.params['repository'], module)
new_pulp_distribution(module.params['repository'], module.params['repository'], repo_href, module)
[new_galaxy_namespace(n, module) for n in module.params['namespaces']]
module.exit_json(changed=True)
if __name__ == '__main__':
main()

View file

@ -127,6 +127,7 @@ def run_module():
'authors': ['Collection author <name@email.com'],
'dependencies': collection['dependencies'],
'license': ['GPL-3.0-or-later'],
'repository': 'https://ansible.com/',
}
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))

View file

@ -1,30 +0,0 @@
- name: Create galaxy_ng namespaces
uri:
url: '{{ pulp_api }}/api/galaxy/v3/_ui/namespaces/'
method: POST
body_format: json
body:
name: '{{ namespace }}'
groups:
- system:partner-engineers
status_code:
- 201
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
loop: '{{ collection_list|map(attribute="namespace")|unique + publish_namespaces }}'
loop_control:
loop_var: namespace
- name: Get galaxy_ng token
uri:
url: '{{ pulp_api }}/api/galaxy/v3/auth/token/'
method: POST
body_format: json
body: {}
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: galaxy_ng_token

View file

@ -14,11 +14,36 @@
- name: run ansible-galaxy collection build tests
import_tasks: build.yml
# The pulp container has a long start up time
# The first task to interact with pulp needs to wait until it responds appropriately
- name: list pulp distributions
uri:
url: '{{ pulp_api }}/pulp/api/v3/distributions/ansible/ansible/'
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distributions
until: pulp_distributions is successful
delay: 1
retries: 60
- name: configure pulp
include_tasks: pulp.yml
- name: configure galaxy_ng
include_tasks: galaxy_ng.yml
- name: Get galaxy_ng token
uri:
url: '{{ galaxy_ng_server }}v3/auth/token/'
method: POST
body_format: json
body: {}
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: galaxy_ng_token
- name: create test ansible.cfg that contains the Galaxy server list
template:
@ -34,19 +59,15 @@
vars:
test_name: '{{ item.name }}'
test_server: '{{ item.server }}'
is_pulp: '{{ item.pulp|default(false) }}'
vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}'
loop:
- name: pulp_v2
server: '{{ pulp_v2_server }}'
pulp: true
- name: pulp_v3
server: '{{ pulp_v3_server }}'
pulp: true
v3: true
- name: galaxy_ng
server: '{{ galaxy_ng_server }}'
pulp: true
v3: true
# We use a module for this so we can speed up the test time.

View file

@ -1,36 +1,9 @@
---
- name: fail to publish with no token - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fail_no_token
failed_when: '"HTTP Code: 401" not in fail_no_token.stderr'
when: not is_pulp
- name: fail to publish with invalid token - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_server }} --token fail {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fail_invalid_token
failed_when: '"HTTP Code: 401" not in fail_invalid_token.stderr'
when: not is_pulp
- name: publish collection - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fallaxy_publish_collection
when: not is_pulp
- name: publish collection - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: pulp_publish_collection
when: is_pulp
- set_fact:
publish_collection: '{{ pulp_publish_collection if pulp_publish_collection is not skipped else fallaxy_publish_collection }}'
register: publish_collection
- name: get result of publish collection - {{ test_name }}
uri:
@ -49,28 +22,12 @@
- publish_collection_actual.json.namespace.name == 'ansible_test'
- publish_collection_actual.json.version == '1.0.0'
- name: fail to publish existing collection version - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fail_publish_existing
failed_when: '"Artifact already exists" not in fail_publish_existing.stderr'
when: not is_pulp
- name: fail to publish existing collection version - {{ test_name }}
command: ansible-galaxy collection publish ansible_test-my_collection-1.0.0.tar.gz -s {{ test_name }} {{ galaxy_verbosity }}
args:
chdir: '{{ galaxy_dir }}'
register: fail_publish_existing
failed_when: fail_publish_existing is not failed
when: is_pulp
- name: reset published collections - {{ test_name }}
uri:
url: '{{ test_server }}custom/reset/'
method: POST
when: not is_pulp
- name: reset published collections - {{ test_name }}
include_tasks: pulp.yml
when: is_pulp

View file

@ -1,165 +1,11 @@
# These tasks configure pulp/pulp_ansible so that we can use the container
# This will also reset pulp between iterations
# The pulp container has a long start up time
# The first task to interact with pulp needs to wait until it responds appropriately
- name: list pulp distributions
uri:
url: '{{ pulp_api }}/pulp/api/v3/distributions/ansible/ansible/'
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distributions
until: pulp_distributions is successful
delay: 1
retries: 60
- when: pulp_distributions.json.count > 0
block:
- name: delete pulp distributions
uri:
url: '{{ pulp_api }}{{ distribution.pulp_href }}'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_delete_tasks
loop: '{{ pulp_distributions.json.results }}'
loop_control:
loop_var: distribution
label: '{{ distribution.name }}'
- name: wait for distribution delete
uri:
url: '{{ pulp_api }}{{ task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_delete
until: pulp_distribution_delete is successful and pulp_distribution_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
loop: '{{ pulp_distribution_delete_tasks.results }}'
loop_control:
loop_var: task
label: '{{ task.json.task }}'
- name: delete pulp repository
uri:
url: '{{ pulp_api }}{{ distribution.repository }}'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repository_delete_tasks
loop: '{{ pulp_distributions.json.results }}'
loop_control:
loop_var: distribution
label: '{{ distribution.name }}'
- name: wait for repository delete
uri:
url: '{{ pulp_api }}{{ task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repository_delete
until: pulp_repository_delete is successful and pulp_repository_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
loop: '{{ pulp_repository_delete_tasks.results }}'
loop_control:
loop_var: task
label: '{{ task.json.task }}'
- name: delete pulp orphans
uri:
url: '{{ pulp_api }}/pulp/api/v3/orphans/'
method: DELETE
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_orphans_delete_task
- name: wait for orphan delete
uri:
url: '{{ pulp_api }}{{ pulp_orphans_delete_task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_orphans_delete
until: pulp_orphans_delete is successful and pulp_orphans_delete.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
- name: create pulp repos
uri:
url: '{{ pulp_api }}/pulp/api/v3/repositories/ansible/ansible/'
method: POST
body_format: json
body:
name: '{{ repo_name }}'
status_code:
- 201
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_repo
loop:
- automation-hub
loop_control:
loop_var: repo_name
- name: create pulp distributions
uri:
url: '{{ pulp_api }}/pulp/api/v3/distributions/ansible/ansible/'
method: POST
body_format: json
body:
name: '{{ repo.repo_name }}'
base_path: '{{ repo.repo_name }}'
repository: '{{ pulp_api }}{{ repo.json.pulp_href }}'
status_code:
- 202
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
register: pulp_distribution_task
loop: '{{ pulp_repo.results }}'
loop_control:
loop_var: repo
- name: wait for distribution creation
uri:
url: '{{ pulp_api }}{{ task.json.task }}'
method: GET
status_code:
- 200
user: '{{ pulp_user }}'
password: '{{ pulp_password }}'
force_basic_auth: true
until: pulp_distribution is successful and pulp_distribution.json.state|default("MISSING") == "completed"
delay: 1
retries: 25
register: pulp_distribution
loop: '{{ pulp_distribution_task.results }}'
loop_control:
loop_var: task
label: '{{ task.json.task }}'
# A module is used to make the tests run quicker as this will send lots of API requests.
- name: reset pulp content
reset_pulp:
pulp_api: '{{ pulp_api }}'
galaxy_ng_server: '{{ galaxy_ng_server }}'
url_username: '{{ pulp_user }}'
url_password: '{{ pulp_password }}'
repository: published
namespaces: '{{ collection_list|map(attribute="namespace")|unique + publish_namespaces }}'

View file

@ -4,7 +4,6 @@ __metaclass__ = type
import os
import tempfile
import uuid
from . import (
CloudProvider,
@ -59,6 +58,27 @@ foreground {
}
'''
# There are 2 overrides here:
# 1. Change the gunicorn bind address from 127.0.0.1 to 0.0.0.0 now that Galaxy NG does not allow us to access the
# Pulp API through it.
# 2. Grant access allowing us to DELETE a namespace in Galaxy NG. This is as CI deletes and recreates repos and
# distributions in Pulp which now breaks the namespace in Galaxy NG. Recreating it is the "simple" fix to get it
# working again.
# These may not be needed in the future, especially if 1 becomes configurable by an env var but for now they must be
# done.
OVERRIDES = b'''#!/usr/bin/execlineb -S0
foreground {
sed -i "0,/\\"127.0.0.1:24817\\"/s//\\"0.0.0.0:24817\\"/" /etc/services.d/pulpcore-api/run
}
# This sed calls changes the first occurrence to "allow" which is conveniently the delete operation for a namespace.
# https://github.com/ansible/galaxy_ng/blob/master/galaxy_ng/app/access_control/statements/standalone.py#L9-L11.
backtick NG_PREFIX { python -c "import galaxy_ng; print(galaxy_ng.__path__[0], end='')" }
importas ng_prefix NG_PREFIX
foreground {
sed -i "0,/\\"effect\\": \\"deny\\"/s//\\"effect\\": \\"allow\\"/" ${ng_prefix}/app/access_control/statements/standalone.py
}'''
class GalaxyProvider(CloudProvider):
"""Galaxy plugin.
@ -76,7 +96,7 @@ class GalaxyProvider(CloudProvider):
self.pulp = os.environ.get(
'ANSIBLE_PULP_CONTAINER',
'docker.io/pulp/pulp-galaxy-ng@sha256:69b4c4cba4908539b56c5592f40d282f938dd1bdf4de5a81e0a8d04ac3e6e326'
'docker.io/pulp/pulp-galaxy-ng@sha256:b79a7be64eff86d8f58db9ca83ed4967bd8b4e45c99addb17a91d11926480cf1'
)
self.containers = []
@ -117,7 +137,8 @@ class GalaxyProvider(CloudProvider):
% ('Using the existing' if p_results else 'Starting a new'),
verbosity=1)
pulp_port = 80
galaxy_port = 80
pulp_port = 24817
if not p_results:
if self.args.docker or container_id:
@ -125,6 +146,7 @@ class GalaxyProvider(CloudProvider):
else:
# publish the simulator ports when not running inside docker
publish_ports = [
'-p', ':'.join((str(galaxy_port),) * 2),
'-p', ':'.join((str(pulp_port),) * 2),
]
@ -140,21 +162,16 @@ class GalaxyProvider(CloudProvider):
pulp_id = stdout.strip()
try:
# Inject our settings.py file
with tempfile.NamedTemporaryFile(delete=False) as settings:
settings.write(SETTINGS)
docker_command(self.args, ['cp', settings.name, '%s:/etc/pulp/settings.py' % pulp_id])
finally:
os.unlink(settings.name)
try:
# Inject our settings.py file
with tempfile.NamedTemporaryFile(delete=False) as admin_pass:
admin_pass.write(SET_ADMIN_PASSWORD)
docker_command(self.args, ['cp', admin_pass.name, '%s:/etc/cont-init.d/111-postgres' % pulp_id])
finally:
os.unlink(admin_pass.name)
injected_files = {
'/etc/pulp/settings.py': SETTINGS,
'/etc/cont-init.d/111-postgres': SET_ADMIN_PASSWORD,
'/etc/cont-init.d/000-ansible-test-overrides': OVERRIDES,
}
for path, content in injected_files.items():
with tempfile.NamedTemporaryFile() as temp_fd:
temp_fd.write(content)
temp_fd.flush()
docker_command(self.args, ['cp', temp_fd.name, '%s:%s' % (pulp_id, path)])
# Start the container
docker_start(self.args, 'ansible-ci-pulp', [])
@ -171,6 +188,7 @@ class GalaxyProvider(CloudProvider):
self._set_cloud_config('PULP_HOST', pulp_host)
self._set_cloud_config('PULP_PORT', str(pulp_port))
self._set_cloud_config('GALAXY_PORT', str(galaxy_port))
self._set_cloud_config('PULP_USER', 'admin')
self._set_cloud_config('PULP_PASSWORD', 'password')
@ -209,22 +227,23 @@ class GalaxyEnvironment(CloudEnvironment):
pulp_user = self._get_cloud_config('PULP_USER')
pulp_password = self._get_cloud_config('PULP_PASSWORD')
pulp_host = self._get_cloud_config('PULP_HOST')
galaxy_port = self._get_cloud_config('GALAXY_PORT')
pulp_port = self._get_cloud_config('PULP_PORT')
return CloudEnvironmentConfig(
ansible_vars=dict(
pulp_user=pulp_user,
pulp_password=pulp_password,
pulp_v2_server='http://%s:%s/pulp_ansible/galaxy/automation-hub/api/' % (pulp_host, pulp_port),
pulp_v3_server='http://%s:%s/pulp_ansible/galaxy/automation-hub/api/' % (pulp_host, pulp_port),
pulp_v2_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
pulp_v3_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
pulp_api='http://%s:%s' % (pulp_host, pulp_port),
galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, pulp_port),
galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
),
env_vars=dict(
PULP_USER=pulp_user,
PULP_PASSWORD=pulp_password,
PULP_V2_SERVER='http://%s:%s/pulp_ansible/galaxy/automation-hub/api/' % (pulp_host, pulp_port),
PULP_V3_SERVER='http://%s:%s/pulp_ansible/galaxy/automation-hub/api/' % (pulp_host, pulp_port),
GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, pulp_port),
PULP_V2_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
PULP_V3_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
),
)