From 00bd0b893d5d21de040b53032c466707bacb3b93 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Mon, 15 Feb 2021 09:45:01 -0500 Subject: [PATCH] ansible-galaxy - set the cache file after getting all collection versions (#73557) * Manage the in-memory cache in _call_galaxy but let the caller set the file cache after getting paginated results * Add a test for caching successful and not caching unsuccessful paginated results Co-authored-by: Sviatoslav Sydorenko --- ...nsible-galaxy-cache-paginated-response.yml | 3 + lib/ansible/galaxy/api.py | 4 +- test/units/galaxy/test_api.py | 150 +++++++++++++++++- 3 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/73557-ansible-galaxy-cache-paginated-response.yml diff --git a/changelogs/fragments/73557-ansible-galaxy-cache-paginated-response.yml b/changelogs/fragments/73557-ansible-galaxy-cache-paginated-response.yml new file mode 100644 index 00000000000..6b95ab11dd4 --- /dev/null +++ b/changelogs/fragments/73557-ansible-galaxy-cache-paginated-response.yml @@ -0,0 +1,3 @@ +bugfixes: + - ansible-galaxy - Cache the responses for available collection versions + after getting all pages. (https://github.com/ansible/ansible/issues/73071) diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index de5d6cc305b..352082ded76 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -389,8 +389,6 @@ class GalaxyAPI: else: path_cache['results'] = data - self._set_cache() - return data def _add_auth_token(self, headers, url, token_type=None, required=False): @@ -759,6 +757,7 @@ class GalaxyAPI: error_context_msg = 'Error when getting collection version metadata for %s.%s:%s from %s (%s)' \ % (namespace, name, version, self.name, self.api_server) data = self._call_galaxy(n_collection_url, error_context_msg=error_context_msg, cache=True) + self._set_cache() return CollectionVersionMetadata(data['namespace']['name'], data['collection']['name'], data['version'], data['download_url'], data['artifact']['sha256'], @@ -843,5 +842,6 @@ class GalaxyAPI: data = self._call_galaxy(to_native(next_link, errors='surrogate_or_strict'), error_context_msg=error_context_msg, cache=True) + self._set_cache() return versions diff --git a/test/units/galaxy/test_api.py b/test/units/galaxy/test_api.py index 1e8af129d6e..ebda706c584 100644 --- a/test/units/galaxy/test_api.py +++ b/test/units/galaxy/test_api.py @@ -63,10 +63,10 @@ def cache_dir(tmp_path_factory, monkeypatch): yield cache_dir -def get_test_galaxy_api(url, version, token_ins=None, token_value=None): +def get_test_galaxy_api(url, version, token_ins=None, token_value=None, no_cache=True): token_value = token_value or "my token" token_ins = token_ins or GalaxyToken(token_value) - api = GalaxyAPI(None, "test", url) + api = GalaxyAPI(None, "test", url, no_cache=no_cache) # Warning, this doesn't test g_connect() because _availabe_api_versions is set here. That means # that urls for v2 servers have to append '/api/' themselves in the input data. api._available_api_versions = {version: '%s' % version} @@ -75,6 +75,60 @@ def get_test_galaxy_api(url, version, token_ins=None, token_value=None): return api +def get_collection_versions(namespace='namespace', name='collection'): + base_url = 'https://galaxy.server.com/api/v2/collections/{0}/{1}/'.format(namespace, name) + versions_url = base_url + 'versions/' + + # Response for collection info + responses = [ + { + "id": 1000, + "href": base_url, + "name": name, + "namespace": { + "id": 30000, + "href": "https://galaxy.ansible.com/api/v1/namespaces/30000/", + "name": namespace, + }, + "versions_url": versions_url, + "latest_version": { + "version": "1.0.5", + "href": versions_url + "1.0.5/" + }, + "deprecated": False, + "created": "2021-02-09T16:55:42.749915-05:00", + "modified": "2021-02-09T16:55:42.749915-05:00", + } + ] + + # Paginated responses for versions + page_versions = (('1.0.0', '1.0.1',), ('1.0.2', '1.0.3',), ('1.0.4', '1.0.5'),) + last_page = None + for page in range(1, len(page_versions) + 1): + if page < len(page_versions): + next_page = versions_url + '?page={0}'.format(page + 1) + else: + next_page = None + + version_results = [] + for version in page_versions[int(page - 1)]: + version_results.append( + {'version': version, 'href': versions_url + '{0}/'.format(version)} + ) + + responses.append( + { + 'count': 6, + 'next': next_page, + 'previous': last_page, + 'results': version_results, + } + ) + last_page = page + + return responses + + def test_api_no_auth(): api = GalaxyAPI(None, "test", "https://galaxy.ansible.com/api/") actual = {} @@ -974,6 +1028,98 @@ def test_cache_invalid_cache_content(content, cache_dir): assert stat.S_IMODE(os.stat(cache_file).st_mode) == 0o664 +def test_cache_complete_pagination(cache_dir, monkeypatch): + + responses = get_collection_versions() + cache_file = os.path.join(cache_dir, 'api.json') + + api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False) + + mock_open = MagicMock( + side_effect=[ + StringIO(to_text(json.dumps(r))) + for r in responses + ] + ) + monkeypatch.setattr(galaxy_api, 'open_url', mock_open) + + actual_versions = api.get_collection_versions('namespace', 'collection') + assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5'] + + with open(cache_file) as fd: + final_cache = json.loads(fd.read()) + + cached_server = final_cache['galaxy.server.com:'] + cached_collection = cached_server['/api/v2/collections/namespace/collection/versions/'] + cached_versions = [r['version'] for r in cached_collection['results']] + + assert final_cache == api._cache + assert cached_versions == actual_versions + + +def test_cache_flaky_pagination(cache_dir, monkeypatch): + + responses = get_collection_versions() + cache_file = os.path.join(cache_dir, 'api.json') + + api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False) + + # First attempt, fail midway through + mock_open = MagicMock( + side_effect=[ + StringIO(to_text(json.dumps(responses[0]))), + StringIO(to_text(json.dumps(responses[1]))), + urllib_error.HTTPError(responses[1]['next'], 500, 'Error', {}, StringIO()), + StringIO(to_text(json.dumps(responses[3]))), + ] + ) + monkeypatch.setattr(galaxy_api, 'open_url', mock_open) + + expected = ( + r'Error when getting available collection versions for namespace\.collection ' + r'from test \(https://galaxy\.server\.com/api/\) ' + r'\(HTTP Code: 500, Message: Error Code: Unknown\)' + ) + with pytest.raises(GalaxyError, match=expected): + api.get_collection_versions('namespace', 'collection') + + with open(cache_file) as fd: + final_cache = json.loads(fd.read()) + + assert final_cache == { + 'version': 1, + 'galaxy.server.com:': { + 'modified': { + 'namespace.collection': responses[0]['modified'] + } + } + } + + # Reset API + api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v2', no_cache=False) + + # Second attempt is successful so cache should be populated + mock_open = MagicMock( + side_effect=[ + StringIO(to_text(json.dumps(r))) + for r in responses + ] + ) + monkeypatch.setattr(galaxy_api, 'open_url', mock_open) + + actual_versions = api.get_collection_versions('namespace', 'collection') + assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5'] + + with open(cache_file) as fd: + final_cache = json.loads(fd.read()) + + cached_server = final_cache['galaxy.server.com:'] + cached_collection = cached_server['/api/v2/collections/namespace/collection/versions/'] + cached_versions = [r['version'] for r in cached_collection['results']] + + assert cached_versions == actual_versions + + def test_world_writable_cache(cache_dir, monkeypatch): mock_warning = MagicMock() monkeypatch.setattr(Display, 'warning', mock_warning)