Support galaxy v3/autohub API in ansible-galaxy (#60982)

* Add galaxy collections API v3 support

Issue: ansible/galaxy-dev#60

- Determine if server supports v3

Use 'available_versions' from `GET /api`
to determine if 'v3' api is available on
the server.

- Support v3 pagination style

ie, 'limit/offset style', with the paginated
responses based on https://jsonapi.org/format/#fetching-pagination

v2 galaxy uses pagination that is more or less
'django rest framework style' or 'page/page_size style',
based on the default drf pagination described
at https://www.django-rest-framework.org/api-guide/pagination/#pagenumberpagination

- Support galaxy v3 style error response

The error objects returned by the galaxy v3 api are based
on the JSONAPI response/errors format
(https://jsonapi.org/format/#errors).

This handles that style response. At least for publish_collection
for now. Needs extracting/generalizing.

Handle HTTPError in CollectionRequirement.from_name()
with _handle_http_error(). It will raise AnsibleError
based on the json in an error response.

- Update unit tests

update test/unit/galaxy/test_collection*
to paramaterize calls to test against
mocked v2 and v3 servers apis.

Update artifacts_versions_json() to tale an
api version paramater.

Add error_json() for generating v3/v3 style error
responses.

So now, the urls generated and the pagination schema
of the response will use the v3 version if
the passed in GalaxyAPI 'galaxy_api' instance
has 'v3' in it's available_api_versions

* Move checking of server avail versions to collections.py

collections.py needs to know the server api versions
supported before it makes collection related calls,
so the 'lazy' server version check in api.GalaxyAPI
is never called and isn't set, so 'v3' servers weren't
found.

Update unit tests to mock the return value of the
request instead of GalaxyAPI itself.
This commit is contained in:
Adrian Likins 2019-08-28 16:59:34 -04:00 committed by GitHub
parent f9b0ab774f
commit af01cb114c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 510 additions and 64 deletions

View file

@ -43,11 +43,14 @@ def g_connect(method):
if not self.initialized:
display.vvvv("Initial connection to galaxy_server: %s" % self.api_server)
server_version = self._get_server_api_version()
if server_version not in self.SUPPORTED_VERSIONS:
raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)
self.baseurl = _urljoin(self.api_server, "api", server_version)
self.version = server_version # for future use
display.vvvv("Base API: %s" % self.baseurl)
self.initialized = True
return method(self, *args, **kwargs)
@ -63,25 +66,32 @@ class GalaxyAPI(object):
SUPPORTED_VERSIONS = ['v1']
def __init__(self, galaxy, name, url, username=None, password=None, token=None):
def __init__(self, galaxy, name, url, username=None, password=None, token=None, token_type=None):
self.galaxy = galaxy
self.name = name
self.username = username
self.password = password
self.token = token
self.token_type = token_type or 'Token'
self.api_server = url
self.validate_certs = not context.CLIARGS['ignore_certs']
self.baseurl = None
self.version = None
self.initialized = False
self.available_api_versions = {}
display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs))
def _auth_header(self, required=True):
def _auth_header(self, required=True, token_type=None):
'''Generate the Authorization header.
Valid token_type values are 'Token' (galaxy v2) and 'Bearer' (galaxy v3)'''
token = self.token.get() if self.token else None
# 'Token' for v2 api, 'Bearer' for v3
token_type = token_type or self.token_type
if token:
return {'Authorization': "Token %s" % token}
return {'Authorization': "%s %s" % (token_type, token)}
elif self.username:
token = "%s:%s" % (to_text(self.username, errors='surrogate_or_strict'),
to_text(self.password, errors='surrogate_or_strict', nonstring='passthru') or '')
@ -123,9 +133,6 @@ class GalaxyAPI(object):
except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
if 'current_version' not in data:
raise AnsibleError("missing required 'current_version' from server response (%s)" % url)
return data['current_version']
@g_connect

View file

@ -310,6 +310,14 @@ class CollectionRequirement:
for api in apis:
collection_url_paths = [api.api_server, 'api', 'v2', 'collections', namespace, name, 'versions']
available_api_versions = get_available_api_versions(api)
if 'v3' in available_api_versions:
# /api/v3/ exists, use it
collection_url_paths[2] = 'v3'
# update this v3 GalaxyAPI to use Bearer token from now on
api.token_type = 'Bearer'
headers = api._auth_header(required=False)
is_single = False
@ -325,10 +333,13 @@ class CollectionRequirement:
try:
resp = json.load(open_url(n_collection_url, validate_certs=api.validate_certs, headers=headers))
except urllib_error.HTTPError as err:
if err.code == 404:
display.vvv("Collection '%s' is not available from server %s %s" % (collection, api.name, api.api_server))
continue
raise
_handle_http_error(err, api, available_api_versions,
'Error fetching info for %s from %s (%s)' % (collection, api.name, api.api_server))
if is_single:
galaxy_info = resp
@ -336,13 +347,24 @@ class CollectionRequirement:
versions = [resp['version']]
else:
versions = []
results_key = 'results'
if 'v3' in available_api_versions:
results_key = 'data'
while True:
# Galaxy supports semver but ansible-galaxy does not. We ignore any versions that don't match
# StrictVersion (x.y.z) and only support pre-releases if an explicit version was set (done above).
versions += [v['version'] for v in resp['results'] if StrictVersion.version_re.match(v['version'])]
if resp['next'] is None:
versions += [v['version'] for v in resp[results_key] if StrictVersion.version_re.match(v['version'])]
next_link = resp.get('next', None)
if 'v3' in available_api_versions:
next_link = resp['links']['next']
if next_link is None:
break
resp = json.load(open_url(to_native(resp['next'], errors='surrogate_or_strict'),
resp = json.load(open_url(to_native(next_link, errors='surrogate_or_strict'),
validate_certs=api.validate_certs, headers=headers))
display.vvv("Collection '%s' obtained from server %s %s" % (collection, api.name, api.api_server))
@ -356,6 +378,45 @@ class CollectionRequirement:
return req
def get_available_api_versions(galaxy_api):
headers = {}
headers.update(galaxy_api._auth_header(required=False))
url = _urljoin(galaxy_api.api_server, "api")
try:
return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
except urllib_error.HTTPError as err:
if err.code != 401:
_handle_http_error(err, galaxy_api, {},
"Error when finding available api versions from %s (%s)" %
(galaxy_api.name, galaxy_api.api_server))
# assume this is v3 and auth is required.
headers = {}
headers.update(galaxy_api._auth_header(token_type='Bearer', required=True))
# try again with auth
try:
return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
except urllib_error.HTTPError as authed_err:
_handle_http_error(authed_err, galaxy_api, {},
"Error when finding available api versions from %s using auth (%s)" %
(galaxy_api.name, galaxy_api.api_server))
except Exception as e:
raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
try:
data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict'))
except Exception as e:
raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
available_versions = data.get('available_versions',
{'v1': '/api/v1',
'v2': '/api/v2'})
return available_versions
def build_collection(collection_path, output_path, force):
"""
Creates the Ansible collection artifact in a .tar.gz file.
@ -410,27 +471,25 @@ def publish_collection(collection_path, api, wait, timeout):
display.display("Publishing collection artifact '%s' to %s %s" % (collection_path, api.name, api.api_server))
n_url = _urljoin(api.api_server, 'api', 'v2', 'collections')
available_api_versions = get_available_api_versions(api)
if 'v3' in available_api_versions:
n_url = _urljoin(api.api_server, 'api', 'v3', 'artifacts', 'collections')
api.token_type = 'Bearer'
headers = {}
headers.update(api._auth_header())
data, content_type = _get_mime_data(b_collection_path)
headers = {
headers.update({
'Content-type': content_type,
'Content-length': len(data),
}
headers.update(api._auth_header())
})
try:
resp = json.load(open_url(n_url, data=data, headers=headers, method='POST', validate_certs=api.validate_certs))
except urllib_error.HTTPError as err:
try:
err_info = json.load(err)
except (AttributeError, ValueError):
err_info = {}
code = to_native(err_info.get('code', 'Unknown'))
message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
raise AnsibleError("Error when publishing collection (HTTP Code: %d, Message: %s Code: %s)"
% (err.code, message, code))
_handle_http_error(err, api, available_api_versions, "Error when publishing collection to %s (%s)" % (api.name, api.api_server))
import_uri = resp['task']
if wait:
@ -1016,3 +1075,33 @@ def _extract_tar_file(tar, filename, b_dest, b_temp_path, expected_hash=None):
os.makedirs(b_parent_dir)
shutil.move(to_bytes(tmpfile_obj.name, errors='surrogate_or_strict'), b_dest_filepath)
def _handle_http_error(http_error, api, available_api_versions, context_error_message):
try:
err_info = json.load(http_error)
except (AttributeError, ValueError):
err_info = {}
if 'v3' in available_api_versions:
message_lines = []
errors = err_info.get('errors', None)
if not errors:
errors = [{'detail': 'Unknown error returned by Galaxy server.',
'code': 'Unknown'}]
for error in errors:
error_msg = error.get('detail') or error.get('title') or 'Unknown error returned by Galaxy server.'
error_code = error.get('code') or 'Unknown'
message_line = "(HTTP Code: %d, Message: %s Code: %s)" % (http_error.code, error_msg, error_code)
message_lines.append(message_line)
full_error_msg = "%s %s" % (context_error_message, ', '.join(message_lines))
else:
code = to_native(err_info.get('code', 'Unknown'))
message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
full_error_msg = "%s (HTTP Code: %d, Message: %s Code: %s)" \
% (context_error_message, http_error.code, message, code)
raise AnsibleError(full_error_msg)

View file

@ -55,6 +55,13 @@ def collection_input(tmp_path_factory):
return collection_dir, output_dir
@pytest.fixture()
def galaxy_api_version(monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
@pytest.fixture()
def collection_artifact(monkeypatch, tmp_path_factory):
''' Creates a temp collection artifact and mocked open_url instance for publishing tests '''
@ -421,6 +428,10 @@ def test_publish_not_a_tarball():
def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
mock_display = MagicMock()
monkeypatch.setattr(Display, 'display', mock_display)
@ -445,12 +456,15 @@ def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
% (artifact_path, galaxy_server.name, galaxy_server.api_server)
assert mock_display.mock_calls[1][1][0] == \
"Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to " \
"--no-wait being set. Import task results can be found at %s"\
% (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
"Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to --no-wait " \
"being set. Import task results can be found at %s" % (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
def test_publish_dont_validate_cert(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
galaxy_server.validate_certs = False
artifact_path, mock_open = collection_artifact
@ -462,29 +476,47 @@ def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
assert mock_open.mock_calls[0][2]['validate_certs'] is False
def test_publish_failure(galaxy_server, collection_artifact):
def test_publish_failure(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
artifact_path, mock_open = collection_artifact
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO())
expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
'server. Code: Unknown)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_with_json_info(galaxy_server, collection_artifact):
def test_publish_failure_with_json_info(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
artifact_path, mock_open = collection_artifact
return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}')
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content)
expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
@pytest.mark.parametrize("api_version,token_type", [
('v2', 'Token'),
('v3', 'Bearer')
])
def test_publish_with_wait(api_version, token_type, galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
mock_display = MagicMock()
monkeypatch.setattr(Display, 'display', mock_display)
@ -501,7 +533,7 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
assert mock_open.call_count == 2
assert mock_open.mock_calls[1][1][0] == fake_import_uri
assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
assert mock_open.mock_calls[1][2]['validate_certs'] is True
assert mock_open.mock_calls[1][2]['method'] == 'GET'
@ -515,7 +547,15 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
'Galaxy server %s %s' % (galaxy_server.name, galaxy_server.api_server)
def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypatch):
@pytest.mark.parametrize("api_version,exp_api_url,token_type", [
('v2', '/api/v2/collections/', 'Token'),
('v3', '/api/v3/artifacts/collections/', 'Bearer')
])
def test_publish_with_wait_timeout(api_version, exp_api_url, token_type, galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
monkeypatch.setattr(time, 'sleep', MagicMock())
mock_vvv = MagicMock()
@ -535,11 +575,11 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
assert mock_open.call_count == 3
assert mock_open.mock_calls[1][1][0] == fake_import_uri
assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
assert mock_open.mock_calls[1][2]['validate_certs'] is True
assert mock_open.mock_calls[1][2]['method'] == 'GET'
assert mock_open.mock_calls[2][1][0] == fake_import_uri
assert mock_open.mock_calls[2][2]['headers']['Authorization'] == 'Token key'
assert mock_open.mock_calls[2][2]['headers']['Authorization'] == '%s key' % token_type
assert mock_open.mock_calls[2][2]['validate_certs'] is True
assert mock_open.mock_calls[2][2]['method'] == 'GET'
@ -549,6 +589,10 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
monkeypatch.setattr(time, 'sleep', MagicMock())
mock_vvv = MagicMock()
@ -588,6 +632,10 @@ def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, m
def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
mock_display = MagicMock()
monkeypatch.setattr(Display, 'display', mock_display)
@ -661,6 +709,10 @@ def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monke
def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
mock_display = MagicMock()
monkeypatch.setattr(Display, 'display', mock_display)
@ -729,6 +781,69 @@ def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_ar
assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error'
def test_publish_failure_v3_with_json_info_409_conflict(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v3': '/api/v3'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
artifact_path, mock_open = collection_artifact
error_response = {
"errors": [
{
"code": "conflict.collection_exists",
"detail": 'Collection "testing-ansible_testing_content-4.0.4" already exists.',
"title": "Conflict.",
"status": "409",
},
]
}
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 409, 'msg', {}, return_content)
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \
' already exists. Code: conflict.collection_exists)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_publish_failure_v3_with_json_info_multiple_errors(galaxy_server, collection_artifact, monkeypatch):
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v3': '/api/v3'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
artifact_path, mock_open = collection_artifact
error_response = {
"errors": [
{
"code": "conflict.collection_exists",
"detail": 'Collection "mynamespace-mycollection-4.1.1" already exists.',
"title": "Conflict.",
"status": "400",
},
{
"code": "quantum_improbability",
"title": "Random(?) quantum improbability.",
"source": {"parameter": "the_arrow_of_time"},
"meta": {"remediation": "Try again before"}
},
]
}
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, return_content)
expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
'(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \
' already exists. Code: conflict.collection_exists),' \
' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)'
with pytest.raises(AnsibleError, match=re.escape(expected)):
collection.publish_collection(artifact_path, galaxy_server, True, 0)
def test_find_existing_collections(tmp_path_factory, monkeypatch):
test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections'))
collection1 = os.path.join(test_dir, 'namespace1', 'collection1')
@ -847,3 +962,91 @@ def test_extract_tar_file_missing_parent_dir(tmp_tarfile):
collection._extract_tar_file(tfile, filename, output_dir, temp_dir, checksum)
os.path.isfile(output_file)
def test_get_available_api_versions_v2_auth_not_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
# mock_avail_ver = MagicMock()
# mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
# monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
response_obj = {
"description": "GALAXY REST API",
"current_version": "v1",
"available_versions": {
"v1": "/api/v1/",
"v2": "/api/v2/"
},
"server_version": "3.2.4",
"version_name": "Doin' it Right",
"team_members": [
"chouseknecht",
"cutwater",
"alikins",
"newswangerd",
"awcrosby",
"tima",
"gregdek"
]
}
artifact_path, mock_open = collection_artifact
return_content = StringIO(to_text(json.dumps(response_obj)))
mock_open.return_value = return_content
res = collection.get_available_api_versions(galaxy_server)
assert res == {'v1': '/api/v1/', 'v2': '/api/v2/'}
def test_get_available_api_versions_v3_auth_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
# mock_avail_ver = MagicMock()
# mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
# monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
error_response = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
artifact_path, mock_open = collection_artifact
return_content = StringIO(to_text(json.dumps(error_response)))
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, return_content)
with pytest.raises(AnsibleError):
collection.get_available_api_versions(galaxy_server)
def test_get_available_api_versions_v3_auth_required_with_auth_on_retry(galaxy_server, collection_artifact, monkeypatch):
# mock_avail_ver = MagicMock()
# mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
# monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
error_obj = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
success_obj = {
"description": "GALAXY REST API",
"current_version": "v1",
"available_versions": {
"v3": "/api/v3/"
},
"server_version": "3.2.4",
"version_name": "Doin' it Right",
"team_members": [
"chouseknecht",
"cutwater",
"alikins",
"newswangerd",
"awcrosby",
"tima",
"gregdek"
]
}
artifact_path, mock_open = collection_artifact
error_response = StringIO(to_text(json.dumps(error_obj)))
success_response = StringIO(to_text(json.dumps(success_obj)))
mock_open.side_effect = [
urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, error_response),
success_response,
]
try:
res = collection.get_available_api_versions(galaxy_server)
except AnsibleError as err:
print(err)
raise
assert res == {'v3': '/api/v3/'}

View file

@ -23,7 +23,7 @@ import ansible.module_utils.six.moves.urllib.error as urllib_error
from ansible import context
from ansible.cli.galaxy import GalaxyCLI
from ansible.errors import AnsibleError
from ansible.galaxy import collection, api, Galaxy
from ansible.galaxy import collection, api
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.utils import context_objects as co
from ansible.utils.display import Display
@ -56,20 +56,57 @@ def artifact_json(namespace, name, version, dependencies, server):
return to_text(json_str)
def artifact_versions_json(namespace, name, versions, server):
def artifact_versions_json(namespace, name, versions, galaxy_api, available_api_versions=None):
results = []
available_api_versions = available_api_versions or {}
api_version = 'v2'
if 'v3' in available_api_versions:
api_version = 'v3'
for version in versions:
results.append({
'href': '%s/api/v2/%s/%s/versions/%s/' % (server, namespace, name, version),
'href': '%s/api/%s/%s/%s/versions/%s/' % (galaxy_api.api_server, api_version, namespace, name, version),
'version': version,
})
json_str = json.dumps({
'count': len(versions),
'next': None,
'previous': None,
'results': results
})
if api_version == 'v2':
json_str = json.dumps({
'count': len(versions),
'next': None,
'previous': None,
'results': results
})
if api_version == 'v3':
response = {'meta': {'count': len(versions)},
'data': results,
'links': {'first': None,
'last': None,
'next': None,
'previous': None},
}
json_str = json.dumps(response)
return to_text(json_str)
def error_json(galaxy_api, errors_to_return=None, available_api_versions=None):
errors_to_return = errors_to_return or []
available_api_versions = available_api_versions or {}
response = {}
api_version = 'v2'
if 'v3' in available_api_versions:
api_version = 'v3'
if api_version == 'v2':
assert len(errors_to_return) <= 1
if errors_to_return:
response = errors_to_return[0]
if api_version == 'v3':
response['errors'] = errors_to_return
json_str = json.dumps(response)
return to_text(json_str)
@ -253,10 +290,20 @@ def test_build_requirement_from_tar_invalid_manifest(tmp_path_factory):
collection.CollectionRequirement.from_tar(tar_path, True, True)
def test_build_requirement_from_name(galaxy_server, monkeypatch):
json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server.api_server)
@pytest.mark.parametrize("api_version,exp_api_url", [
('v2', '/api/v2/collections/namespace/collection/versions/'),
('v3', '/api/v3/collections/namespace/collection/versions/')
])
def test_build_requirement_from_name(api_version, exp_api_url, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
avail_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = avail_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server, avail_api_versions)
mock_open = MagicMock()
mock_open.return_value = StringIO(json_str)
monkeypatch.setattr(collection, 'open_url', mock_open)
actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '*', True, True)
@ -271,13 +318,22 @@ def test_build_requirement_from_name(galaxy_server, monkeypatch):
assert actual.dependencies is None
assert mock_open.call_count == 1
assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" % galaxy_server.api_server
assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch):
@pytest.mark.parametrize("api_version,exp_api_url", [
('v2', '/api/v2/collections/namespace/collection/versions/'),
('v3', '/api/v3/collections/namespace/collection/versions/')
])
def test_build_requirement_from_name_with_prerelease(api_version, exp_api_url, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
avail_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = avail_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '2.0.1-beta.1', '2.0.1'],
galaxy_server.api_server)
galaxy_server, avail_api_versions)
mock_open = MagicMock()
mock_open.return_value = StringIO(json_str)
@ -295,15 +351,25 @@ def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch)
assert actual.dependencies is None
assert mock_open.call_count == 1
assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \
% galaxy_server.api_server
assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monkeypatch):
@pytest.mark.parametrize("api_version,exp_api_url", [
('v2', '/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/'),
('v3', '/api/v3/collections/namespace/collection/versions/2.0.1-beta.1/')
])
def test_build_requirment_from_name_with_prerelease_explicit(api_version, exp_api_url, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
avail_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = avail_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = artifact_json('namespace', 'collection', '2.0.1-beta.1', {}, galaxy_server.api_server)
mock_open = MagicMock()
mock_open.return_value = StringIO(json_str)
mock_open.side_effect = (
StringIO(json_str),
)
monkeypatch.setattr(collection, 'open_url', mock_open)
@ -320,13 +386,21 @@ def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monk
assert actual.dependencies == {}
assert mock_open.call_count == 1
assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/" \
% galaxy_server.api_server
assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch):
json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server.api_server)
@pytest.mark.parametrize("api_version,exp_api_url", [
('v2', '/api/v2/collections/namespace/collection/versions/'),
('v3', '/api/v3/collections/namespace/collection/versions/')
])
def test_build_requirement_from_name_second_server(api_version, exp_api_url, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
avail_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = avail_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server, avail_api_versions)
mock_open = MagicMock()
mock_open.side_effect = (
urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, None),
@ -343,17 +417,15 @@ def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch):
assert actual.namespace == u'namespace'
assert actual.name == u'collection'
assert actual.b_path is None
assert actual.api == galaxy_server
# assert actual.api == galaxy_server
assert actual.skip is False
assert actual.versions == set([u'1.0.2', u'1.0.3'])
assert actual.latest_version == u'1.0.3'
assert actual.dependencies is None
assert mock_open.call_count == 2
assert mock_open.mock_calls[0][1][0] == u"https://broken.com/api/v2/collections/namespace/collection/versions/"
assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
assert mock_open.mock_calls[1][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \
% galaxy_server.api_server
assert mock_open.mock_calls[0][1][0] == u"https://broken.com%s" % exp_api_url
assert mock_open.mock_calls[1][1][0] == u"%s%s" % (galaxy_server.api_server, exp_api_url)
assert mock_open.mock_calls[1][2] == {'validate_certs': True, "headers": {}}
@ -363,12 +435,75 @@ def test_build_requirement_from_name_missing(galaxy_server, monkeypatch):
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
expected = "Failed to find collection namespace.collection:*"
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name('namespace.collection',
[galaxy_server, galaxy_server], '*', False, True)
@pytest.mark.parametrize("api_version,errors_to_return,expected", [
('v2',
[],
'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
('v2',
[{'message': 'Polarization error. Try flipping it over.', 'code': 'polarization_error'}],
'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Polarization error. Try flipping it over. Code: polarization_error\\)'),
('v3',
[],
'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
('v3',
[{'code': 'invalid_param', 'detail': '"easy" is not a valid query param'}],
'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: "easy" is not a valid query param Code: invalid_param\\)'),
])
def test_build_requirement_from_name_400_bad_request(api_version, errors_to_return, expected, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
available_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = available_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions)
mock_open = MagicMock()
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, StringIO(json_str))
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name('namespace.collection',
[galaxy_server, galaxy_server], '*', False)
@pytest.mark.parametrize("api_version,errors_to_return,expected", [
('v2',
[],
'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
('v3',
[],
'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
('v3',
[{'code': 'unauthorized', 'detail': 'The request was not authorized'}],
'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: The request was not authorized Code: unauthorized\\)'),
])
def test_build_requirement_from_name_401_unauthorized(api_version, errors_to_return, expected, galaxy_server, monkeypatch):
mock_avail_ver = MagicMock()
available_api_versions = {api_version: '/api/%s' % api_version}
mock_avail_ver.return_value = available_api_versions
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions)
mock_open = MagicMock()
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO(json_str))
with pytest.raises(AnsibleError, match=expected):
collection.CollectionRequirement.from_name('namespace.collection',
[galaxy_server, galaxy_server], '*', False)
def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
json_str = artifact_json('namespace', 'collection', '2.0.0', {}, galaxy_server.api_server)
mock_open = MagicMock()
@ -376,6 +511,10 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '2.0.0', True, True)
assert actual.namespace == u'namespace'
@ -395,13 +534,17 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
def test_build_requirement_from_name_multiple_versions_one_match(galaxy_server, monkeypatch):
json_str1 = artifact_versions_json('namespace', 'collection', ['2.0.0', '2.0.1', '2.0.2'],
galaxy_server.api_server)
galaxy_server)
json_str2 = artifact_json('namespace', 'collection', '2.0.1', {}, galaxy_server.api_server)
mock_open = MagicMock()
mock_open.side_effect = (StringIO(json_str1), StringIO(json_str2))
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '>=2.0.1,<2.0.2',
True, True)
@ -467,6 +610,10 @@ def test_build_requirement_from_name_multiple_version_results(galaxy_server, mon
monkeypatch.setattr(collection, 'open_url', mock_open)
mock_avail_ver = MagicMock()
mock_avail_ver.return_value = {'v2': '/api/v2'}
monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '!=2.0.2',
True, True)