diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 7df9d3b1afa..45de4cbb895 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -43,11 +43,14 @@ def g_connect(method): if not self.initialized: display.vvvv("Initial connection to galaxy_server: %s" % self.api_server) server_version = self._get_server_api_version() + if server_version not in self.SUPPORTED_VERSIONS: raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version) self.baseurl = _urljoin(self.api_server, "api", server_version) + self.version = server_version # for future use + display.vvvv("Base API: %s" % self.baseurl) self.initialized = True return method(self, *args, **kwargs) @@ -63,25 +66,32 @@ class GalaxyAPI(object): SUPPORTED_VERSIONS = ['v1'] - def __init__(self, galaxy, name, url, username=None, password=None, token=None): + def __init__(self, galaxy, name, url, username=None, password=None, token=None, token_type=None): self.galaxy = galaxy self.name = name self.username = username self.password = password self.token = token + self.token_type = token_type or 'Token' self.api_server = url self.validate_certs = not context.CLIARGS['ignore_certs'] self.baseurl = None self.version = None self.initialized = False + self.available_api_versions = {} display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs)) - def _auth_header(self, required=True): + def _auth_header(self, required=True, token_type=None): + '''Generate the Authorization header. + + Valid token_type values are 'Token' (galaxy v2) and 'Bearer' (galaxy v3)''' token = self.token.get() if self.token else None + # 'Token' for v2 api, 'Bearer' for v3 + token_type = token_type or self.token_type if token: - return {'Authorization': "Token %s" % token} + return {'Authorization': "%s %s" % (token_type, token)} elif self.username: token = "%s:%s" % (to_text(self.username, errors='surrogate_or_strict'), to_text(self.password, errors='surrogate_or_strict', nonstring='passthru') or '') @@ -123,9 +133,6 @@ class GalaxyAPI(object): except Exception as e: raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e))) - if 'current_version' not in data: - raise AnsibleError("missing required 'current_version' from server response (%s)" % url) - return data['current_version'] @g_connect diff --git a/lib/ansible/galaxy/collection.py b/lib/ansible/galaxy/collection.py index 65f2e2e3e4d..58c0a25db9d 100644 --- a/lib/ansible/galaxy/collection.py +++ b/lib/ansible/galaxy/collection.py @@ -310,6 +310,14 @@ class CollectionRequirement: for api in apis: collection_url_paths = [api.api_server, 'api', 'v2', 'collections', namespace, name, 'versions'] + + available_api_versions = get_available_api_versions(api) + if 'v3' in available_api_versions: + # /api/v3/ exists, use it + collection_url_paths[2] = 'v3' + # update this v3 GalaxyAPI to use Bearer token from now on + api.token_type = 'Bearer' + headers = api._auth_header(required=False) is_single = False @@ -325,10 +333,13 @@ class CollectionRequirement: try: resp = json.load(open_url(n_collection_url, validate_certs=api.validate_certs, headers=headers)) except urllib_error.HTTPError as err: + if err.code == 404: display.vvv("Collection '%s' is not available from server %s %s" % (collection, api.name, api.api_server)) continue - raise + + _handle_http_error(err, api, available_api_versions, + 'Error fetching info for %s from %s (%s)' % (collection, api.name, api.api_server)) if is_single: galaxy_info = resp @@ -336,13 +347,24 @@ class CollectionRequirement: versions = [resp['version']] else: versions = [] + + results_key = 'results' + if 'v3' in available_api_versions: + results_key = 'data' + while True: # Galaxy supports semver but ansible-galaxy does not. We ignore any versions that don't match # StrictVersion (x.y.z) and only support pre-releases if an explicit version was set (done above). - versions += [v['version'] for v in resp['results'] if StrictVersion.version_re.match(v['version'])] - if resp['next'] is None: + versions += [v['version'] for v in resp[results_key] if StrictVersion.version_re.match(v['version'])] + + next_link = resp.get('next', None) + if 'v3' in available_api_versions: + next_link = resp['links']['next'] + + if next_link is None: break - resp = json.load(open_url(to_native(resp['next'], errors='surrogate_or_strict'), + + resp = json.load(open_url(to_native(next_link, errors='surrogate_or_strict'), validate_certs=api.validate_certs, headers=headers)) display.vvv("Collection '%s' obtained from server %s %s" % (collection, api.name, api.api_server)) @@ -356,6 +378,45 @@ class CollectionRequirement: return req +def get_available_api_versions(galaxy_api): + headers = {} + headers.update(galaxy_api._auth_header(required=False)) + + url = _urljoin(galaxy_api.api_server, "api") + try: + return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs) + except urllib_error.HTTPError as err: + if err.code != 401: + _handle_http_error(err, galaxy_api, {}, + "Error when finding available api versions from %s (%s)" % + (galaxy_api.name, galaxy_api.api_server)) + + # assume this is v3 and auth is required. + headers = {} + headers.update(galaxy_api._auth_header(token_type='Bearer', required=True)) + # try again with auth + try: + return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs) + except urllib_error.HTTPError as authed_err: + _handle_http_error(authed_err, galaxy_api, {}, + "Error when finding available api versions from %s using auth (%s)" % + (galaxy_api.name, galaxy_api.api_server)) + + except Exception as e: + raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e))) + + try: + data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict')) + except Exception as e: + raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e))) + + available_versions = data.get('available_versions', + {'v1': '/api/v1', + 'v2': '/api/v2'}) + + return available_versions + + def build_collection(collection_path, output_path, force): """ Creates the Ansible collection artifact in a .tar.gz file. @@ -410,27 +471,25 @@ def publish_collection(collection_path, api, wait, timeout): display.display("Publishing collection artifact '%s' to %s %s" % (collection_path, api.name, api.api_server)) n_url = _urljoin(api.api_server, 'api', 'v2', 'collections') + available_api_versions = get_available_api_versions(api) + + if 'v3' in available_api_versions: + n_url = _urljoin(api.api_server, 'api', 'v3', 'artifacts', 'collections') + api.token_type = 'Bearer' + + headers = {} + headers.update(api._auth_header()) data, content_type = _get_mime_data(b_collection_path) - headers = { + headers.update({ 'Content-type': content_type, 'Content-length': len(data), - } - headers.update(api._auth_header()) + }) try: resp = json.load(open_url(n_url, data=data, headers=headers, method='POST', validate_certs=api.validate_certs)) except urllib_error.HTTPError as err: - try: - err_info = json.load(err) - except (AttributeError, ValueError): - err_info = {} - - code = to_native(err_info.get('code', 'Unknown')) - message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.')) - - raise AnsibleError("Error when publishing collection (HTTP Code: %d, Message: %s Code: %s)" - % (err.code, message, code)) + _handle_http_error(err, api, available_api_versions, "Error when publishing collection to %s (%s)" % (api.name, api.api_server)) import_uri = resp['task'] if wait: @@ -1016,3 +1075,33 @@ def _extract_tar_file(tar, filename, b_dest, b_temp_path, expected_hash=None): os.makedirs(b_parent_dir) shutil.move(to_bytes(tmpfile_obj.name, errors='surrogate_or_strict'), b_dest_filepath) + + +def _handle_http_error(http_error, api, available_api_versions, context_error_message): + try: + err_info = json.load(http_error) + except (AttributeError, ValueError): + err_info = {} + + if 'v3' in available_api_versions: + message_lines = [] + errors = err_info.get('errors', None) + + if not errors: + errors = [{'detail': 'Unknown error returned by Galaxy server.', + 'code': 'Unknown'}] + + for error in errors: + error_msg = error.get('detail') or error.get('title') or 'Unknown error returned by Galaxy server.' + error_code = error.get('code') or 'Unknown' + message_line = "(HTTP Code: %d, Message: %s Code: %s)" % (http_error.code, error_msg, error_code) + message_lines.append(message_line) + + full_error_msg = "%s %s" % (context_error_message, ', '.join(message_lines)) + else: + code = to_native(err_info.get('code', 'Unknown')) + message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.')) + full_error_msg = "%s (HTTP Code: %d, Message: %s Code: %s)" \ + % (context_error_message, http_error.code, message, code) + + raise AnsibleError(full_error_msg) diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py index 8d0aff3c868..6276fb346cc 100644 --- a/test/units/galaxy/test_collection.py +++ b/test/units/galaxy/test_collection.py @@ -55,6 +55,13 @@ def collection_input(tmp_path_factory): return collection_dir, output_dir +@pytest.fixture() +def galaxy_api_version(monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + @pytest.fixture() def collection_artifact(monkeypatch, tmp_path_factory): ''' Creates a temp collection artifact and mocked open_url instance for publishing tests ''' @@ -421,6 +428,10 @@ def test_publish_not_a_tarball(): def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) @@ -445,12 +456,15 @@ def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch): assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \ % (artifact_path, galaxy_server.name, galaxy_server.api_server) assert mock_display.mock_calls[1][1][0] == \ - "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to " \ - "--no-wait being set. Import task results can be found at %s"\ - % (galaxy_server.name, galaxy_server.api_server, fake_import_uri) + "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to --no-wait " \ + "being set. Import task results can be found at %s" % (galaxy_server.name, galaxy_server.api_server, fake_import_uri) -def test_publish_dont_validate_cert(galaxy_server, collection_artifact): +def test_publish_dont_validate_cert(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + galaxy_server.validate_certs = False artifact_path, mock_open = collection_artifact @@ -462,29 +476,47 @@ def test_publish_dont_validate_cert(galaxy_server, collection_artifact): assert mock_open.mock_calls[0][2]['validate_certs'] is False -def test_publish_failure(galaxy_server, collection_artifact): +def test_publish_failure(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + artifact_path, mock_open = collection_artifact mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO()) - expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \ + expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ + '(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \ 'server. Code: Unknown)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0) -def test_publish_failure_with_json_info(galaxy_server, collection_artifact): +def test_publish_failure_with_json_info(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + artifact_path, mock_open = collection_artifact return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}') mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content) - expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)' + expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ + '(HTTP Code: 503, Message: Galaxy error message Code: GWE002)' with pytest.raises(AnsibleError, match=re.escape(expected)): collection.publish_collection(artifact_path, galaxy_server, True, 0) -def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch): +@pytest.mark.parametrize("api_version,token_type", [ + ('v2', 'Token'), + ('v3', 'Bearer') +]) +def test_publish_with_wait(api_version, token_type, galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {api_version: '/api/%s' % api_version} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) @@ -501,7 +533,7 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch): assert mock_open.call_count == 2 assert mock_open.mock_calls[1][1][0] == fake_import_uri - assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key' + assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type assert mock_open.mock_calls[1][2]['validate_certs'] is True assert mock_open.mock_calls[1][2]['method'] == 'GET' @@ -515,7 +547,15 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch): 'Galaxy server %s %s' % (galaxy_server.name, galaxy_server.api_server) -def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypatch): +@pytest.mark.parametrize("api_version,exp_api_url,token_type", [ + ('v2', '/api/v2/collections/', 'Token'), + ('v3', '/api/v3/artifacts/collections/', 'Bearer') +]) +def test_publish_with_wait_timeout(api_version, exp_api_url, token_type, galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {api_version: '/api/%s' % api_version} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + monkeypatch.setattr(time, 'sleep', MagicMock()) mock_vvv = MagicMock() @@ -535,11 +575,11 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat assert mock_open.call_count == 3 assert mock_open.mock_calls[1][1][0] == fake_import_uri - assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key' + assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type assert mock_open.mock_calls[1][2]['validate_certs'] is True assert mock_open.mock_calls[1][2]['method'] == 'GET' assert mock_open.mock_calls[2][1][0] == fake_import_uri - assert mock_open.mock_calls[2][2]['headers']['Authorization'] == 'Token key' + assert mock_open.mock_calls[2][2]['headers']['Authorization'] == '%s key' % token_type assert mock_open.mock_calls[2][2]['validate_certs'] is True assert mock_open.mock_calls[2][2]['method'] == 'GET' @@ -549,6 +589,10 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + monkeypatch.setattr(time, 'sleep', MagicMock()) mock_vvv = MagicMock() @@ -588,6 +632,10 @@ def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, m def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) @@ -661,6 +709,10 @@ def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monke def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) @@ -729,6 +781,69 @@ def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_ar assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error' +def test_publish_failure_v3_with_json_info_409_conflict(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v3': '/api/v3'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + artifact_path, mock_open = collection_artifact + + error_response = { + "errors": [ + { + "code": "conflict.collection_exists", + "detail": 'Collection "testing-ansible_testing_content-4.0.4" already exists.', + "title": "Conflict.", + "status": "409", + }, + ] + } + + return_content = StringIO(to_text(json.dumps(error_response))) + mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 409, 'msg', {}, return_content) + + expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ + '(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \ + ' already exists. Code: conflict.collection_exists)' + with pytest.raises(AnsibleError, match=re.escape(expected)): + collection.publish_collection(artifact_path, galaxy_server, True, 0) + + +def test_publish_failure_v3_with_json_info_multiple_errors(galaxy_server, collection_artifact, monkeypatch): + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v3': '/api/v3'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + artifact_path, mock_open = collection_artifact + + error_response = { + "errors": [ + { + "code": "conflict.collection_exists", + "detail": 'Collection "mynamespace-mycollection-4.1.1" already exists.', + "title": "Conflict.", + "status": "400", + }, + { + "code": "quantum_improbability", + "title": "Random(?) quantum improbability.", + "source": {"parameter": "the_arrow_of_time"}, + "meta": {"remediation": "Try again before"} + }, + ] + } + + return_content = StringIO(to_text(json.dumps(error_response))) + mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, return_content) + + expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \ + '(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \ + ' already exists. Code: conflict.collection_exists),' \ + ' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)' + with pytest.raises(AnsibleError, match=re.escape(expected)): + collection.publish_collection(artifact_path, galaxy_server, True, 0) + + def test_find_existing_collections(tmp_path_factory, monkeypatch): test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections')) collection1 = os.path.join(test_dir, 'namespace1', 'collection1') @@ -847,3 +962,91 @@ def test_extract_tar_file_missing_parent_dir(tmp_tarfile): collection._extract_tar_file(tfile, filename, output_dir, temp_dir, checksum) os.path.isfile(output_file) + + +def test_get_available_api_versions_v2_auth_not_required_without_auth(galaxy_server, collection_artifact, monkeypatch): + # mock_avail_ver = MagicMock() + # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version} + # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + response_obj = { + "description": "GALAXY REST API", + "current_version": "v1", + "available_versions": { + "v1": "/api/v1/", + "v2": "/api/v2/" + }, + "server_version": "3.2.4", + "version_name": "Doin' it Right", + "team_members": [ + "chouseknecht", + "cutwater", + "alikins", + "newswangerd", + "awcrosby", + "tima", + "gregdek" + ] + } + + artifact_path, mock_open = collection_artifact + + return_content = StringIO(to_text(json.dumps(response_obj))) + mock_open.return_value = return_content + res = collection.get_available_api_versions(galaxy_server) + + assert res == {'v1': '/api/v1/', 'v2': '/api/v2/'} + + +def test_get_available_api_versions_v3_auth_required_without_auth(galaxy_server, collection_artifact, monkeypatch): + # mock_avail_ver = MagicMock() + # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version} + # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + error_response = {'code': 'unauthorized', 'detail': 'The request was not authorized'} + artifact_path, mock_open = collection_artifact + + return_content = StringIO(to_text(json.dumps(error_response))) + mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, return_content) + with pytest.raises(AnsibleError): + collection.get_available_api_versions(galaxy_server) + + +def test_get_available_api_versions_v3_auth_required_with_auth_on_retry(galaxy_server, collection_artifact, monkeypatch): + # mock_avail_ver = MagicMock() + # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version} + # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + error_obj = {'code': 'unauthorized', 'detail': 'The request was not authorized'} + success_obj = { + "description": "GALAXY REST API", + "current_version": "v1", + "available_versions": { + "v3": "/api/v3/" + }, + "server_version": "3.2.4", + "version_name": "Doin' it Right", + "team_members": [ + "chouseknecht", + "cutwater", + "alikins", + "newswangerd", + "awcrosby", + "tima", + "gregdek" + ] + } + + artifact_path, mock_open = collection_artifact + + error_response = StringIO(to_text(json.dumps(error_obj))) + success_response = StringIO(to_text(json.dumps(success_obj))) + mock_open.side_effect = [ + urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, error_response), + success_response, + ] + + try: + res = collection.get_available_api_versions(galaxy_server) + except AnsibleError as err: + print(err) + raise + + assert res == {'v3': '/api/v3/'} diff --git a/test/units/galaxy/test_collection_install.py b/test/units/galaxy/test_collection_install.py index 06b75836136..1f5b3858073 100644 --- a/test/units/galaxy/test_collection_install.py +++ b/test/units/galaxy/test_collection_install.py @@ -23,7 +23,7 @@ import ansible.module_utils.six.moves.urllib.error as urllib_error from ansible import context from ansible.cli.galaxy import GalaxyCLI from ansible.errors import AnsibleError -from ansible.galaxy import collection, api, Galaxy +from ansible.galaxy import collection, api from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.utils import context_objects as co from ansible.utils.display import Display @@ -56,20 +56,57 @@ def artifact_json(namespace, name, version, dependencies, server): return to_text(json_str) -def artifact_versions_json(namespace, name, versions, server): +def artifact_versions_json(namespace, name, versions, galaxy_api, available_api_versions=None): results = [] + available_api_versions = available_api_versions or {} + api_version = 'v2' + if 'v3' in available_api_versions: + api_version = 'v3' for version in versions: results.append({ - 'href': '%s/api/v2/%s/%s/versions/%s/' % (server, namespace, name, version), + 'href': '%s/api/%s/%s/%s/versions/%s/' % (galaxy_api.api_server, api_version, namespace, name, version), 'version': version, }) - json_str = json.dumps({ - 'count': len(versions), - 'next': None, - 'previous': None, - 'results': results - }) + if api_version == 'v2': + json_str = json.dumps({ + 'count': len(versions), + 'next': None, + 'previous': None, + 'results': results + }) + + if api_version == 'v3': + response = {'meta': {'count': len(versions)}, + 'data': results, + 'links': {'first': None, + 'last': None, + 'next': None, + 'previous': None}, + } + json_str = json.dumps(response) + return to_text(json_str) + + +def error_json(galaxy_api, errors_to_return=None, available_api_versions=None): + errors_to_return = errors_to_return or [] + available_api_versions = available_api_versions or {} + + response = {} + + api_version = 'v2' + if 'v3' in available_api_versions: + api_version = 'v3' + + if api_version == 'v2': + assert len(errors_to_return) <= 1 + if errors_to_return: + response = errors_to_return[0] + + if api_version == 'v3': + response['errors'] = errors_to_return + + json_str = json.dumps(response) return to_text(json_str) @@ -253,10 +290,20 @@ def test_build_requirement_from_tar_invalid_manifest(tmp_path_factory): collection.CollectionRequirement.from_tar(tar_path, True, True) -def test_build_requirement_from_name(galaxy_server, monkeypatch): - json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server.api_server) +@pytest.mark.parametrize("api_version,exp_api_url", [ + ('v2', '/api/v2/collections/namespace/collection/versions/'), + ('v3', '/api/v3/collections/namespace/collection/versions/') +]) +def test_build_requirement_from_name(api_version, exp_api_url, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + avail_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = avail_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server, avail_api_versions) mock_open = MagicMock() mock_open.return_value = StringIO(json_str) + monkeypatch.setattr(collection, 'open_url', mock_open) actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '*', True, True) @@ -271,13 +318,22 @@ def test_build_requirement_from_name(galaxy_server, monkeypatch): assert actual.dependencies is None assert mock_open.call_count == 1 - assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" % galaxy_server.api_server + assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url) assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}} -def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch): +@pytest.mark.parametrize("api_version,exp_api_url", [ + ('v2', '/api/v2/collections/namespace/collection/versions/'), + ('v3', '/api/v3/collections/namespace/collection/versions/') +]) +def test_build_requirement_from_name_with_prerelease(api_version, exp_api_url, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + avail_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = avail_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '2.0.1-beta.1', '2.0.1'], - galaxy_server.api_server) + galaxy_server, avail_api_versions) mock_open = MagicMock() mock_open.return_value = StringIO(json_str) @@ -295,15 +351,25 @@ def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch) assert actual.dependencies is None assert mock_open.call_count == 1 - assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \ - % galaxy_server.api_server + assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url) assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}} -def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monkeypatch): +@pytest.mark.parametrize("api_version,exp_api_url", [ + ('v2', '/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/'), + ('v3', '/api/v3/collections/namespace/collection/versions/2.0.1-beta.1/') +]) +def test_build_requirment_from_name_with_prerelease_explicit(api_version, exp_api_url, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + avail_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = avail_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + json_str = artifact_json('namespace', 'collection', '2.0.1-beta.1', {}, galaxy_server.api_server) mock_open = MagicMock() - mock_open.return_value = StringIO(json_str) + mock_open.side_effect = ( + StringIO(json_str), + ) monkeypatch.setattr(collection, 'open_url', mock_open) @@ -320,13 +386,21 @@ def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monk assert actual.dependencies == {} assert mock_open.call_count == 1 - assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/" \ - % galaxy_server.api_server + assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url) assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}} -def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch): - json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server.api_server) +@pytest.mark.parametrize("api_version,exp_api_url", [ + ('v2', '/api/v2/collections/namespace/collection/versions/'), + ('v3', '/api/v3/collections/namespace/collection/versions/') +]) +def test_build_requirement_from_name_second_server(api_version, exp_api_url, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + avail_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = avail_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server, avail_api_versions) mock_open = MagicMock() mock_open.side_effect = ( urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, None), @@ -343,17 +417,15 @@ def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch): assert actual.namespace == u'namespace' assert actual.name == u'collection' assert actual.b_path is None - assert actual.api == galaxy_server + # assert actual.api == galaxy_server assert actual.skip is False assert actual.versions == set([u'1.0.2', u'1.0.3']) assert actual.latest_version == u'1.0.3' assert actual.dependencies is None assert mock_open.call_count == 2 - assert mock_open.mock_calls[0][1][0] == u"https://broken.com/api/v2/collections/namespace/collection/versions/" - assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}} - assert mock_open.mock_calls[1][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \ - % galaxy_server.api_server + assert mock_open.mock_calls[0][1][0] == u"https://broken.com%s" % exp_api_url + assert mock_open.mock_calls[1][1][0] == u"%s%s" % (galaxy_server.api_server, exp_api_url) assert mock_open.mock_calls[1][2] == {'validate_certs': True, "headers": {}} @@ -363,12 +435,75 @@ def test_build_requirement_from_name_missing(galaxy_server, monkeypatch): monkeypatch.setattr(collection, 'open_url', mock_open) + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + expected = "Failed to find collection namespace.collection:*" with pytest.raises(AnsibleError, match=expected): collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server, galaxy_server], '*', False, True) +@pytest.mark.parametrize("api_version,errors_to_return,expected", [ + ('v2', + [], + 'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'), + ('v2', + [{'message': 'Polarization error. Try flipping it over.', 'code': 'polarization_error'}], + 'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Polarization error. Try flipping it over. Code: polarization_error\\)'), + ('v3', + [], + 'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'), + ('v3', + [{'code': 'invalid_param', 'detail': '"easy" is not a valid query param'}], + 'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: "easy" is not a valid query param Code: invalid_param\\)'), +]) +def test_build_requirement_from_name_400_bad_request(api_version, errors_to_return, expected, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + available_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = available_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions) + + mock_open = MagicMock() + monkeypatch.setattr(collection, 'open_url', mock_open) + mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, StringIO(json_str)) + + with pytest.raises(AnsibleError, match=expected): + collection.CollectionRequirement.from_name('namespace.collection', + [galaxy_server, galaxy_server], '*', False) + + +@pytest.mark.parametrize("api_version,errors_to_return,expected", [ + ('v2', + [], + 'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'), + ('v3', + [], + 'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'), + ('v3', + [{'code': 'unauthorized', 'detail': 'The request was not authorized'}], + 'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: The request was not authorized Code: unauthorized\\)'), +]) +def test_build_requirement_from_name_401_unauthorized(api_version, errors_to_return, expected, galaxy_server, monkeypatch): + mock_avail_ver = MagicMock() + available_api_versions = {api_version: '/api/%s' % api_version} + mock_avail_ver.return_value = available_api_versions + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + + json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions) + + mock_open = MagicMock() + monkeypatch.setattr(collection, 'open_url', mock_open) + mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO(json_str)) + + with pytest.raises(AnsibleError, match=expected): + collection.CollectionRequirement.from_name('namespace.collection', + [galaxy_server, galaxy_server], '*', False) + + def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch): json_str = artifact_json('namespace', 'collection', '2.0.0', {}, galaxy_server.api_server) mock_open = MagicMock() @@ -376,6 +511,10 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch): monkeypatch.setattr(collection, 'open_url', mock_open) + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '2.0.0', True, True) assert actual.namespace == u'namespace' @@ -395,13 +534,17 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch): def test_build_requirement_from_name_multiple_versions_one_match(galaxy_server, monkeypatch): json_str1 = artifact_versions_json('namespace', 'collection', ['2.0.0', '2.0.1', '2.0.2'], - galaxy_server.api_server) + galaxy_server) json_str2 = artifact_json('namespace', 'collection', '2.0.1', {}, galaxy_server.api_server) mock_open = MagicMock() mock_open.side_effect = (StringIO(json_str1), StringIO(json_str2)) monkeypatch.setattr(collection, 'open_url', mock_open) + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '>=2.0.1,<2.0.2', True, True) @@ -467,6 +610,10 @@ def test_build_requirement_from_name_multiple_version_results(galaxy_server, mon monkeypatch.setattr(collection, 'open_url', mock_open) + mock_avail_ver = MagicMock() + mock_avail_ver.return_value = {'v2': '/api/v2'} + monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver) + actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '!=2.0.2', True, True)