ansible-galaxy - add download option (#67632)

* ansible-galaxy - add download option

* Fix sanity issues and added integration tests

* Fix doc suggestions

* Added --pre option

changelogs/fragments/galaxy-download.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+- ansible-galaxy - Add ``download`` option for ``ansible-galaxy collection`` to download collections and their dependencies for an offline install

docs/docsite/rst/user_guide/collections_using.rst

@@ -55,6 +55,60 @@ Configuring the ``ansible-galaxy`` client
 
 .. include:: ../shared_snippets/galaxy_server_list.txt
 
+.. _collections_downloading:
+
+Downloading collections
+=======================
+
+To download a collection and its dependencies for an offline install, run ``ansible-galaxy collection download``. This
+downloads the collections specified and their dependencies to the specified folder and creates a ``requirements.yml``
+file which can be used to install those collections on a host without access to a Galaxy server. All the collections
+are downloaded by default to the ``./collections`` folder.
+
+Just like the ``install`` command, the collections are sourced based on the
+:ref:`configured galaxy server config <galaxy_server_config>`. Even if a collection to download was specified by a URL
+or path to a tarball, the collection will be redownloaded from the configured Galaxy server.
+
+Collections can be specified as one or multiple collections or with a ``requirements.yml`` file just like
+``ansible-galaxy collection install``.
+
+To download a single collection and its dependencies:
+
+.. code-block:: bash
+
+   ansible-galaxy collection download my_namespace.my_collection
+
+To download a single collection at a specific version:
+
+.. code-block:: bash
+
+   ansible-galaxy collection download my_namespace.my_collection:1.0.0
+
+To download multiple collections either specify multiple collections as command line arguments as shown above or use a
+requirements file in the format documented with :ref:`collection_requirements_file`.
+
+.. code-block:: bash
+
+   ansible-galaxy collection download -r requirements.yml
+
+All the collections are downloaded by default to the ``./collections`` folder but you can use ``-p`` or
+``--download-path`` to specify another path:
+
+.. code-block:: bash
+
+   ansible-galaxy collection download my_namespace.my_collection -p ~/offline-collections
+
+Once you have downloaded the collections, the folder contains the collections specified, their dependencies, and a
+``requirements.yml`` file. You can use this folder as is with ``ansible-galaxy collection install`` to install the
+collections on a host without access to a Galaxy or Automation Hub server.
+
+.. code-block:: bash
+
+   # This must be run from the folder that contains the offline collections and requirements.yml file downloaded
+   # by the internet-connected host
+   cd ~/offline-collections
+   ansible-galaxy collection install -r requirements.yml
+
 .. _collections_listing:
 
 Listing collections

lib/ansible/cli/galaxy.py

@@ -25,6 +25,7 @@ from ansible.galaxy.api import GalaxyAPI
 from ansible.galaxy.collection import (
     build_collection,
     CollectionRequirement,
+    download_collections,
     find_existing_collections,
     install_collections,
     publish_collection,
@@ -162,6 +163,7 @@ class GalaxyCLI(CLI):
         collection = type_parser.add_parser('collection', help='Manage an Ansible Galaxy collection.')
         collection_parser = collection.add_subparsers(metavar='COLLECTION_ACTION', dest='action')
         collection_parser.required = True
+        self.add_download_options(collection_parser, parents=[common])
         self.add_init_options(collection_parser, parents=[common, force])
         self.add_build_options(collection_parser, parents=[common, force])
         self.add_publish_options(collection_parser, parents=[common])
@@ -184,6 +186,25 @@ class GalaxyCLI(CLI):
         self.add_info_options(role_parser, parents=[common, roles_path, offline])
         self.add_install_options(role_parser, parents=[common, force, roles_path])
 
+    def add_download_options(self, parser, parents=None):
+        download_parser = parser.add_parser('download', parents=parents,
+                                            help='Download collections and their dependencies as a tarball for an '
+                                                 'offline install.')
+        download_parser.set_defaults(func=self.execute_download)
+
+        download_parser.add_argument('args', help='Collection(s)', metavar='collection', nargs='*')
+
+        download_parser.add_argument('-n', '--no-deps', dest='no_deps', action='store_true', default=False,
+                                     help="Don't download collection(s) listed as dependencies.")
+
+        download_parser.add_argument('-p', '--download-path', dest='download_path',
+                                     default='./collections',
+                                     help='The directory to download the collections to.')
+        download_parser.add_argument('-r', '--requirements-file', dest='requirements',
+                                     help='A file containing a list of collections to be downloaded.')
+        download_parser.add_argument('--pre', dest='allow_pre_release', action='store_true',
+                                     help='Include pre-release versions. Semantic versioning pre-releases are ignored by default')
+
     def add_init_options(self, parser, parents=None):
         galaxy_type = 'collection' if parser.metavar == 'COLLECTION_ACTION' else 'role'
 
@@ -714,6 +735,28 @@ class GalaxyCLI(CLI):
             collection_path = GalaxyCLI._resolve_path(collection_path)
             build_collection(collection_path, output_path, force)
 
+    def execute_download(self):
+        collections = context.CLIARGS['args']
+        no_deps = context.CLIARGS['no_deps']
+        download_path = context.CLIARGS['download_path']
+        ignore_certs = context.CLIARGS['ignore_certs']
+
+        requirements_file = context.CLIARGS['requirements']
+        if requirements_file:
+            requirements_file = GalaxyCLI._resolve_path(requirements_file)
+
+        requirements = self._require_one_of_collections_requirements(collections, requirements_file)
+
+        download_path = GalaxyCLI._resolve_path(download_path)
+        b_download_path = to_bytes(download_path, errors='surrogate_or_strict')
+        if not os.path.exists(b_download_path):
+            os.makedirs(b_download_path)
+
+        download_collections(requirements, download_path, self.api_servers, (not ignore_certs), no_deps,
+                             context.CLIARGS['allow_pre_release'])
+
+        return 0
+
     def execute_init(self):
         """
         Creates the skeleton framework of a role or collection that complies with the Galaxy metadata format.

lib/ansible/galaxy/collection.py

@@ -178,6 +178,17 @@ class CollectionRequirement:
 
         self.versions = new_versions
 
+    def download(self, b_path):
+        download_url = self._metadata.download_url
+        artifact_hash = self._metadata.artifact_sha256
+        headers = {}
+        self.api._add_auth_token(headers, download_url, required=False)
+
+        b_collection_path = _download_file(download_url, b_path, artifact_hash, self.api.validate_certs,
+                                           headers=headers)
+
+        return to_text(b_collection_path, errors='surrogate_or_strict')
+
     def install(self, path, b_temp_path):
         if self.skip:
             display.display("Skipping '%s' as it is already installed" % to_text(self))
@@ -189,13 +200,7 @@ class CollectionRequirement:
         display.display("Installing '%s:%s' to '%s'" % (to_text(self), self.latest_version, collection_path))
 
         if self.b_path is None:
-            download_url = self._metadata.download_url
+            self.b_path = self.download(b_temp_path)
-            artifact_hash = self._metadata.artifact_sha256
-            headers = {}
-            self.api._add_auth_token(headers, download_url, required=False)
-
-            self.b_path = _download_file(download_url, b_temp_path, artifact_hash, self.api.validate_certs,
-                                         headers=headers)
 
         if os.path.exists(b_collection_path):
             shutil.rmtree(b_collection_path)
@@ -493,6 +498,43 @@ def build_collection(collection_path, output_path, force):
     _build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest)
 
 
+def download_collections(collections, output_path, apis, validate_certs, no_deps, allow_pre_release):
+    """
+    Download Ansible collections as their tarball from a Galaxy server to the path specified and creates a requirements
+    file of the downloaded requirements to be used for an install.
+
+    :param collections: The collections to download, should be a list of tuples with (name, requirement, Galaxy Server).
+    :param output_path: The path to download the collections to.
+    :param apis: A list of GalaxyAPIs to query when search for a collection.
+    :param validate_certs: Whether to validate the certificate if downloading a tarball from a non-Galaxy host.
+    :param no_deps: Ignore any collection dependencies and only download the base requirements.
+    :param allow_pre_release: Do not ignore pre-release versions when selecting the latest.
+    """
+    with _tempdir() as b_temp_path:
+        display.display("Process install dependency map")
+        with _display_progress():
+            dep_map = _build_dependency_map(collections, [], b_temp_path, apis, validate_certs, True, True, no_deps,
+                                            allow_pre_release=allow_pre_release)
+
+        requirements = []
+        display.display("Starting collection download process to '%s'" % output_path)
+        with _display_progress():
+            for name, requirement in dep_map.items():
+                collection_filename = "%s-%s-%s.tar.gz" % (requirement.namespace, requirement.name,
+                                                           requirement.latest_version)
+                dest_path = os.path.join(output_path, collection_filename)
+                requirements.append({'name': collection_filename, 'version': requirement.latest_version})
+
+                display.display("Downloading collection '%s' to '%s'" % (name, dest_path))
+                b_temp_download_path = requirement.download(b_temp_path)
+                shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict'))
+
+            requirements_path = os.path.join(output_path, 'requirements.yml')
+            display.display("Writing requirements.yml file of downloaded collections to '%s'" % requirements_path)
+            with open(to_bytes(requirements_path, errors='surrogate_or_strict'), mode='wb') as req_fd:
+                req_fd.write(to_bytes(yaml.safe_dump({'collections': requirements}), errors='surrogate_or_strict'))
+
+
 def publish_collection(collection_path, api, wait, timeout):
     """
     Publish an Ansible collection tarball into an Ansible Galaxy server.

test/integration/targets/ansible-galaxy-collection/tasks/download.yml

@@ -0,0 +1,88 @@
+---
+- name: create test download dir
+  file:
+    path: '{{ galaxy_dir }}/download'
+    state: directory
+
+- name: download collection with multiple dependencides
+  command: ansible-galaxy collection download parent_dep.parent_collection -s {{ fallaxy_galaxy_server }}
+  register: download_collection
+  args:
+    chdir: '{{ galaxy_dir }}/download'
+
+- name: get result of download collection with multiple dependencies
+  find:
+    path: '{{ galaxy_dir }}/download/collections'
+    file_type: file
+  register: download_collection_actual
+
+- name: assert download collection with multiple dependencies
+  assert:
+    that:
+    - '"Downloading collection ''parent_dep.parent_collection'' to" in download_collection.stdout'
+    - '"Downloading collection ''child_dep.child_collection'' to" in download_collection.stdout'
+    - '"Downloading collection ''child_dep.child_dep2'' to" in download_collection.stdout'
+    - download_collection_actual.examined == 4
+    - download_collection_actual.matched == 4
+    - (download_collection_actual.files[0].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
+    - (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
+    - (download_collection_actual.files[2].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
+    - (download_collection_actual.files[3].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
+
+- name: test install of download requirements file
+  command: ansible-galaxy collection install -r requirements.yml -p '{{ galaxy_dir }}/download'
+  args:
+    chdir: '{{ galaxy_dir }}/download/collections'
+  register: install_download
+
+- name: get result of test install of download requirements file
+  slurp:
+    path: '{{ galaxy_dir }}/download/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json'
+  register: install_download_actual
+  loop_control:
+    loop_var: collection
+  loop:
+  - namespace: parent_dep
+    name: parent_collection
+  - namespace: child_dep
+    name: child_collection
+  - namespace: child_dep
+    name: child_dep2
+
+- name: assert test install of download requirements file
+  assert:
+    that:
+    - '"Installing ''parent_dep.parent_collection:1.0.0'' to" in install_download.stdout'
+    - '"Installing ''child_dep.child_collection:0.9.9'' to" in install_download.stdout'
+    - '"Installing ''child_dep.child_dep2:1.2.2'' to" in install_download.stdout'
+    - (install_download_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0'
+    - (install_download_actual.results[1].content | b64decode | from_json).collection_info.version == '0.9.9'
+    - (install_download_actual.results[2].content | b64decode | from_json).collection_info.version == '1.2.2'
+
+- name: create test requirements file for download
+  copy:
+    content: |
+      collections:
+      - name: namespace1.name1
+        version: 1.1.0-beta.1
+
+    dest: '{{ galaxy_dir }}/download/download.yml'
+
+- name: download collection with req to custom dir
+  command: ansible-galaxy collection download -r '{{ galaxy_dir }}/download/download.yml' -s {{ fallaxy_ah_server }} -p '{{ galaxy_dir }}/download/collections-custom'
+  register: download_req_custom_path
+
+- name: get result of download collection with req to custom dir
+  find:
+    path: '{{ galaxy_dir }}/download/collections-custom'
+    file_type: file
+  register: download_req_custom_path_actual
+
+- name: assert download collection with multiple dependencies
+  assert:
+    that:
+    - '"Downloading collection ''namespace1.name1'' to" in download_req_custom_path.stdout'
+    - download_req_custom_path_actual.examined == 2
+    - download_req_custom_path_actual.matched == 2
+    - (download_req_custom_path_actual.files[0].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz']
+    - (download_req_custom_path_actual.files[1].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz']

test/integration/targets/ansible-galaxy-collection/tasks/main.yml

@@ -31,7 +31,7 @@
     server: '{{ fallaxy_ah_server }}'
 
 # We use a module for this so we can speed up the test time.
-- name: setup test collections for install test
+- name: setup test collections for install and download test
   setup_collections:
     server: '{{ fallaxy_galaxy_server }}'
     token: '{{ fallaxy_token }}'
@@ -148,3 +148,6 @@
     server: '{{ fallaxy_galaxy_server }}'
   - name: automation_hub
     server: '{{ fallaxy_ah_server }}'
+
+- name: run ansible-galaxy collection download tests
+  include_tasks: download.yml