From a2deeb8fa27633194d12dfd8e8768ab57100e6d1 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Wed, 25 Mar 2020 08:32:43 +1000 Subject: [PATCH] ansible-galaxy - add download option (#67632) * ansible-galaxy - add download option * Fix sanity issues and added integration tests * Fix doc suggestions * Added --pre option --- changelogs/fragments/galaxy-download.yaml | 2 + .../rst/user_guide/collections_using.rst | 54 ++++++++++++ lib/ansible/cli/galaxy.py | 43 +++++++++ lib/ansible/galaxy/collection.py | 56 ++++++++++-- .../tasks/download.yml | 88 +++++++++++++++++++ .../ansible-galaxy-collection/tasks/main.yml | 5 +- 6 files changed, 240 insertions(+), 8 deletions(-) create mode 100644 changelogs/fragments/galaxy-download.yaml create mode 100644 test/integration/targets/ansible-galaxy-collection/tasks/download.yml diff --git a/changelogs/fragments/galaxy-download.yaml b/changelogs/fragments/galaxy-download.yaml new file mode 100644 index 00000000000..2887c3d09e8 --- /dev/null +++ b/changelogs/fragments/galaxy-download.yaml @@ -0,0 +1,2 @@ +minor_changes: +- ansible-galaxy - Add ``download`` option for ``ansible-galaxy collection`` to download collections and their dependencies for an offline install diff --git a/docs/docsite/rst/user_guide/collections_using.rst b/docs/docsite/rst/user_guide/collections_using.rst index 1d0632b3407..1ea1b1acf37 100644 --- a/docs/docsite/rst/user_guide/collections_using.rst +++ b/docs/docsite/rst/user_guide/collections_using.rst @@ -55,6 +55,60 @@ Configuring the ``ansible-galaxy`` client .. include:: ../shared_snippets/galaxy_server_list.txt +.. _collections_downloading: + +Downloading collections +======================= + +To download a collection and its dependencies for an offline install, run ``ansible-galaxy collection download``. This +downloads the collections specified and their dependencies to the specified folder and creates a ``requirements.yml`` +file which can be used to install those collections on a host without access to a Galaxy server. All the collections +are downloaded by default to the ``./collections`` folder. + +Just like the ``install`` command, the collections are sourced based on the +:ref:`configured galaxy server config `. Even if a collection to download was specified by a URL +or path to a tarball, the collection will be redownloaded from the configured Galaxy server. + +Collections can be specified as one or multiple collections or with a ``requirements.yml`` file just like +``ansible-galaxy collection install``. + +To download a single collection and its dependencies: + +.. code-block:: bash + + ansible-galaxy collection download my_namespace.my_collection + +To download a single collection at a specific version: + +.. code-block:: bash + + ansible-galaxy collection download my_namespace.my_collection:1.0.0 + +To download multiple collections either specify multiple collections as command line arguments as shown above or use a +requirements file in the format documented with :ref:`collection_requirements_file`. + +.. code-block:: bash + + ansible-galaxy collection download -r requirements.yml + +All the collections are downloaded by default to the ``./collections`` folder but you can use ``-p`` or +``--download-path`` to specify another path: + +.. code-block:: bash + + ansible-galaxy collection download my_namespace.my_collection -p ~/offline-collections + +Once you have downloaded the collections, the folder contains the collections specified, their dependencies, and a +``requirements.yml`` file. You can use this folder as is with ``ansible-galaxy collection install`` to install the +collections on a host without access to a Galaxy or Automation Hub server. + +.. code-block:: bash + + # This must be run from the folder that contains the offline collections and requirements.yml file downloaded + # by the internet-connected host + cd ~/offline-collections + ansible-galaxy collection install -r requirements.yml + .. _collections_listing: Listing collections diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 96422b22fac..89a9dcc547d 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -25,6 +25,7 @@ from ansible.galaxy.api import GalaxyAPI from ansible.galaxy.collection import ( build_collection, CollectionRequirement, + download_collections, find_existing_collections, install_collections, publish_collection, @@ -162,6 +163,7 @@ class GalaxyCLI(CLI): collection = type_parser.add_parser('collection', help='Manage an Ansible Galaxy collection.') collection_parser = collection.add_subparsers(metavar='COLLECTION_ACTION', dest='action') collection_parser.required = True + self.add_download_options(collection_parser, parents=[common]) self.add_init_options(collection_parser, parents=[common, force]) self.add_build_options(collection_parser, parents=[common, force]) self.add_publish_options(collection_parser, parents=[common]) @@ -184,6 +186,25 @@ class GalaxyCLI(CLI): self.add_info_options(role_parser, parents=[common, roles_path, offline]) self.add_install_options(role_parser, parents=[common, force, roles_path]) + def add_download_options(self, parser, parents=None): + download_parser = parser.add_parser('download', parents=parents, + help='Download collections and their dependencies as a tarball for an ' + 'offline install.') + download_parser.set_defaults(func=self.execute_download) + + download_parser.add_argument('args', help='Collection(s)', metavar='collection', nargs='*') + + download_parser.add_argument('-n', '--no-deps', dest='no_deps', action='store_true', default=False, + help="Don't download collection(s) listed as dependencies.") + + download_parser.add_argument('-p', '--download-path', dest='download_path', + default='./collections', + help='The directory to download the collections to.') + download_parser.add_argument('-r', '--requirements-file', dest='requirements', + help='A file containing a list of collections to be downloaded.') + download_parser.add_argument('--pre', dest='allow_pre_release', action='store_true', + help='Include pre-release versions. Semantic versioning pre-releases are ignored by default') + def add_init_options(self, parser, parents=None): galaxy_type = 'collection' if parser.metavar == 'COLLECTION_ACTION' else 'role' @@ -714,6 +735,28 @@ class GalaxyCLI(CLI): collection_path = GalaxyCLI._resolve_path(collection_path) build_collection(collection_path, output_path, force) + def execute_download(self): + collections = context.CLIARGS['args'] + no_deps = context.CLIARGS['no_deps'] + download_path = context.CLIARGS['download_path'] + ignore_certs = context.CLIARGS['ignore_certs'] + + requirements_file = context.CLIARGS['requirements'] + if requirements_file: + requirements_file = GalaxyCLI._resolve_path(requirements_file) + + requirements = self._require_one_of_collections_requirements(collections, requirements_file) + + download_path = GalaxyCLI._resolve_path(download_path) + b_download_path = to_bytes(download_path, errors='surrogate_or_strict') + if not os.path.exists(b_download_path): + os.makedirs(b_download_path) + + download_collections(requirements, download_path, self.api_servers, (not ignore_certs), no_deps, + context.CLIARGS['allow_pre_release']) + + return 0 + def execute_init(self): """ Creates the skeleton framework of a role or collection that complies with the Galaxy metadata format. diff --git a/lib/ansible/galaxy/collection.py b/lib/ansible/galaxy/collection.py index 1b7a258df6e..d571ff455ee 100644 --- a/lib/ansible/galaxy/collection.py +++ b/lib/ansible/galaxy/collection.py @@ -178,6 +178,17 @@ class CollectionRequirement: self.versions = new_versions + def download(self, b_path): + download_url = self._metadata.download_url + artifact_hash = self._metadata.artifact_sha256 + headers = {} + self.api._add_auth_token(headers, download_url, required=False) + + b_collection_path = _download_file(download_url, b_path, artifact_hash, self.api.validate_certs, + headers=headers) + + return to_text(b_collection_path, errors='surrogate_or_strict') + def install(self, path, b_temp_path): if self.skip: display.display("Skipping '%s' as it is already installed" % to_text(self)) @@ -189,13 +200,7 @@ class CollectionRequirement: display.display("Installing '%s:%s' to '%s'" % (to_text(self), self.latest_version, collection_path)) if self.b_path is None: - download_url = self._metadata.download_url - artifact_hash = self._metadata.artifact_sha256 - headers = {} - self.api._add_auth_token(headers, download_url, required=False) - - self.b_path = _download_file(download_url, b_temp_path, artifact_hash, self.api.validate_certs, - headers=headers) + self.b_path = self.download(b_temp_path) if os.path.exists(b_collection_path): shutil.rmtree(b_collection_path) @@ -493,6 +498,43 @@ def build_collection(collection_path, output_path, force): _build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest) +def download_collections(collections, output_path, apis, validate_certs, no_deps, allow_pre_release): + """ + Download Ansible collections as their tarball from a Galaxy server to the path specified and creates a requirements + file of the downloaded requirements to be used for an install. + + :param collections: The collections to download, should be a list of tuples with (name, requirement, Galaxy Server). + :param output_path: The path to download the collections to. + :param apis: A list of GalaxyAPIs to query when search for a collection. + :param validate_certs: Whether to validate the certificate if downloading a tarball from a non-Galaxy host. + :param no_deps: Ignore any collection dependencies and only download the base requirements. + :param allow_pre_release: Do not ignore pre-release versions when selecting the latest. + """ + with _tempdir() as b_temp_path: + display.display("Process install dependency map") + with _display_progress(): + dep_map = _build_dependency_map(collections, [], b_temp_path, apis, validate_certs, True, True, no_deps, + allow_pre_release=allow_pre_release) + + requirements = [] + display.display("Starting collection download process to '%s'" % output_path) + with _display_progress(): + for name, requirement in dep_map.items(): + collection_filename = "%s-%s-%s.tar.gz" % (requirement.namespace, requirement.name, + requirement.latest_version) + dest_path = os.path.join(output_path, collection_filename) + requirements.append({'name': collection_filename, 'version': requirement.latest_version}) + + display.display("Downloading collection '%s' to '%s'" % (name, dest_path)) + b_temp_download_path = requirement.download(b_temp_path) + shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict')) + + requirements_path = os.path.join(output_path, 'requirements.yml') + display.display("Writing requirements.yml file of downloaded collections to '%s'" % requirements_path) + with open(to_bytes(requirements_path, errors='surrogate_or_strict'), mode='wb') as req_fd: + req_fd.write(to_bytes(yaml.safe_dump({'collections': requirements}), errors='surrogate_or_strict')) + + def publish_collection(collection_path, api, wait, timeout): """ Publish an Ansible collection tarball into an Ansible Galaxy server. diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml new file mode 100644 index 00000000000..f5bfe69d194 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml @@ -0,0 +1,88 @@ +--- +- name: create test download dir + file: + path: '{{ galaxy_dir }}/download' + state: directory + +- name: download collection with multiple dependencides + command: ansible-galaxy collection download parent_dep.parent_collection -s {{ fallaxy_galaxy_server }} + register: download_collection + args: + chdir: '{{ galaxy_dir }}/download' + +- name: get result of download collection with multiple dependencies + find: + path: '{{ galaxy_dir }}/download/collections' + file_type: file + register: download_collection_actual + +- name: assert download collection with multiple dependencies + assert: + that: + - '"Downloading collection ''parent_dep.parent_collection'' to" in download_collection.stdout' + - '"Downloading collection ''child_dep.child_collection'' to" in download_collection.stdout' + - '"Downloading collection ''child_dep.child_dep2'' to" in download_collection.stdout' + - download_collection_actual.examined == 4 + - download_collection_actual.matched == 4 + - (download_collection_actual.files[0].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz'] + - (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz'] + - (download_collection_actual.files[2].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz'] + - (download_collection_actual.files[3].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz'] + +- name: test install of download requirements file + command: ansible-galaxy collection install -r requirements.yml -p '{{ galaxy_dir }}/download' + args: + chdir: '{{ galaxy_dir }}/download/collections' + register: install_download + +- name: get result of test install of download requirements file + slurp: + path: '{{ galaxy_dir }}/download/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json' + register: install_download_actual + loop_control: + loop_var: collection + loop: + - namespace: parent_dep + name: parent_collection + - namespace: child_dep + name: child_collection + - namespace: child_dep + name: child_dep2 + +- name: assert test install of download requirements file + assert: + that: + - '"Installing ''parent_dep.parent_collection:1.0.0'' to" in install_download.stdout' + - '"Installing ''child_dep.child_collection:0.9.9'' to" in install_download.stdout' + - '"Installing ''child_dep.child_dep2:1.2.2'' to" in install_download.stdout' + - (install_download_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' + - (install_download_actual.results[1].content | b64decode | from_json).collection_info.version == '0.9.9' + - (install_download_actual.results[2].content | b64decode | from_json).collection_info.version == '1.2.2' + +- name: create test requirements file for download + copy: + content: | + collections: + - name: namespace1.name1 + version: 1.1.0-beta.1 + + dest: '{{ galaxy_dir }}/download/download.yml' + +- name: download collection with req to custom dir + command: ansible-galaxy collection download -r '{{ galaxy_dir }}/download/download.yml' -s {{ fallaxy_ah_server }} -p '{{ galaxy_dir }}/download/collections-custom' + register: download_req_custom_path + +- name: get result of download collection with req to custom dir + find: + path: '{{ galaxy_dir }}/download/collections-custom' + file_type: file + register: download_req_custom_path_actual + +- name: assert download collection with multiple dependencies + assert: + that: + - '"Downloading collection ''namespace1.name1'' to" in download_req_custom_path.stdout' + - download_req_custom_path_actual.examined == 2 + - download_req_custom_path_actual.matched == 2 + - (download_req_custom_path_actual.files[0].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz'] + - (download_req_custom_path_actual.files[1].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz'] diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml index 6dc2ab1c01a..0a34acc65ee 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -31,7 +31,7 @@ server: '{{ fallaxy_ah_server }}' # We use a module for this so we can speed up the test time. -- name: setup test collections for install test +- name: setup test collections for install and download test setup_collections: server: '{{ fallaxy_galaxy_server }}' token: '{{ fallaxy_token }}' @@ -148,3 +148,6 @@ server: '{{ fallaxy_galaxy_server }}' - name: automation_hub server: '{{ fallaxy_ah_server }}' + +- name: run ansible-galaxy collection download tests + include_tasks: download.yml