ansible-galaxy - add download option (#67632)
* ansible-galaxy - add download option * Fix sanity issues and added integration tests * Fix doc suggestions * Added --pre option
This commit is contained in:
Jordan Borean
GitHub
parent
28f8b89760
commit
a2deeb8fa2
6 changed files with 240 additions and 8 deletions
2
changelogs/fragments/galaxy-download.yaml
Normal file
2
changelogs/fragments/galaxy-download.yaml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
minor_changes:
|
||||
- ansible-galaxy - Add ``download`` option for ``ansible-galaxy collection`` to download collections and their dependencies for an offline install
|
||||
|
|
@ -55,6 +55,60 @@ Configuring the ``ansible-galaxy`` client
|
|||
|
||||
.. include:: ../shared_snippets/galaxy_server_list.txt
|
||||
|
||||
.. _collections_downloading:
|
||||
|
||||
Downloading collections
|
||||
=======================
|
||||
|
||||
To download a collection and its dependencies for an offline install, run ``ansible-galaxy collection download``. This
|
||||
downloads the collections specified and their dependencies to the specified folder and creates a ``requirements.yml``
|
||||
file which can be used to install those collections on a host without access to a Galaxy server. All the collections
|
||||
are downloaded by default to the ``./collections`` folder.
|
||||
|
||||
Just like the ``install`` command, the collections are sourced based on the
|
||||
:ref:`configured galaxy server config <galaxy_server_config>`. Even if a collection to download was specified by a URL
|
||||
or path to a tarball, the collection will be redownloaded from the configured Galaxy server.
|
||||
|
||||
Collections can be specified as one or multiple collections or with a ``requirements.yml`` file just like
|
||||
``ansible-galaxy collection install``.
|
||||
|
||||
To download a single collection and its dependencies:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ansible-galaxy collection download my_namespace.my_collection
|
||||
|
||||
To download a single collection at a specific version:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ansible-galaxy collection download my_namespace.my_collection:1.0.0
|
||||
|
||||
To download multiple collections either specify multiple collections as command line arguments as shown above or use a
|
||||
requirements file in the format documented with :ref:`collection_requirements_file`.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ansible-galaxy collection download -r requirements.yml
|
||||
|
||||
All the collections are downloaded by default to the ``./collections`` folder but you can use ``-p`` or
|
||||
``--download-path`` to specify another path:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ansible-galaxy collection download my_namespace.my_collection -p ~/offline-collections
|
||||
|
||||
Once you have downloaded the collections, the folder contains the collections specified, their dependencies, and a
|
||||
``requirements.yml`` file. You can use this folder as is with ``ansible-galaxy collection install`` to install the
|
||||
collections on a host without access to a Galaxy or Automation Hub server.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# This must be run from the folder that contains the offline collections and requirements.yml file downloaded
|
||||
# by the internet-connected host
|
||||
cd ~/offline-collections
|
||||
ansible-galaxy collection install -r requirements.yml
|
||||
|
||||
.. _collections_listing:
|
||||
|
||||
Listing collections
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from ansible.galaxy.api import GalaxyAPI
|
|||
from ansible.galaxy.collection import (
|
||||
build_collection,
|
||||
CollectionRequirement,
|
||||
download_collections,
|
||||
find_existing_collections,
|
||||
install_collections,
|
||||
publish_collection,
|
||||
|
|
@ -162,6 +163,7 @@ class GalaxyCLI(CLI):
|
|||
collection = type_parser.add_parser('collection', help='Manage an Ansible Galaxy collection.')
|
||||
collection_parser = collection.add_subparsers(metavar='COLLECTION_ACTION', dest='action')
|
||||
collection_parser.required = True
|
||||
self.add_download_options(collection_parser, parents=[common])
|
||||
self.add_init_options(collection_parser, parents=[common, force])
|
||||
self.add_build_options(collection_parser, parents=[common, force])
|
||||
self.add_publish_options(collection_parser, parents=[common])
|
||||
|
|
@ -184,6 +186,25 @@ class GalaxyCLI(CLI):
|
|||
self.add_info_options(role_parser, parents=[common, roles_path, offline])
|
||||
self.add_install_options(role_parser, parents=[common, force, roles_path])
|
||||
|
||||
def add_download_options(self, parser, parents=None):
|
||||
download_parser = parser.add_parser('download', parents=parents,
|
||||
help='Download collections and their dependencies as a tarball for an '
|
||||
'offline install.')
|
||||
download_parser.set_defaults(func=self.execute_download)
|
||||
|
||||
download_parser.add_argument('args', help='Collection(s)', metavar='collection', nargs='*')
|
||||
|
||||
download_parser.add_argument('-n', '--no-deps', dest='no_deps', action='store_true', default=False,
|
||||
help="Don't download collection(s) listed as dependencies.")
|
||||
|
||||
download_parser.add_argument('-p', '--download-path', dest='download_path',
|
||||
default='./collections',
|
||||
help='The directory to download the collections to.')
|
||||
download_parser.add_argument('-r', '--requirements-file', dest='requirements',
|
||||
help='A file containing a list of collections to be downloaded.')
|
||||
download_parser.add_argument('--pre', dest='allow_pre_release', action='store_true',
|
||||
help='Include pre-release versions. Semantic versioning pre-releases are ignored by default')
|
||||
|
||||
def add_init_options(self, parser, parents=None):
|
||||
galaxy_type = 'collection' if parser.metavar == 'COLLECTION_ACTION' else 'role'
|
||||
|
||||
|
|
@ -714,6 +735,28 @@ class GalaxyCLI(CLI):
|
|||
collection_path = GalaxyCLI._resolve_path(collection_path)
|
||||
build_collection(collection_path, output_path, force)
|
||||
|
||||
def execute_download(self):
|
||||
collections = context.CLIARGS['args']
|
||||
no_deps = context.CLIARGS['no_deps']
|
||||
download_path = context.CLIARGS['download_path']
|
||||
ignore_certs = context.CLIARGS['ignore_certs']
|
||||
|
||||
requirements_file = context.CLIARGS['requirements']
|
||||
if requirements_file:
|
||||
requirements_file = GalaxyCLI._resolve_path(requirements_file)
|
||||
|
||||
requirements = self._require_one_of_collections_requirements(collections, requirements_file)
|
||||
|
||||
download_path = GalaxyCLI._resolve_path(download_path)
|
||||
b_download_path = to_bytes(download_path, errors='surrogate_or_strict')
|
||||
if not os.path.exists(b_download_path):
|
||||
os.makedirs(b_download_path)
|
||||
|
||||
download_collections(requirements, download_path, self.api_servers, (not ignore_certs), no_deps,
|
||||
context.CLIARGS['allow_pre_release'])
|
||||
|
||||
return 0
|
||||
|
||||
def execute_init(self):
|
||||
"""
|
||||
Creates the skeleton framework of a role or collection that complies with the Galaxy metadata format.
|
||||
|
|
|
|||
|
|
@ -178,6 +178,17 @@ class CollectionRequirement:
|
|||
|
||||
self.versions = new_versions
|
||||
|
||||
def download(self, b_path):
|
||||
download_url = self._metadata.download_url
|
||||
artifact_hash = self._metadata.artifact_sha256
|
||||
headers = {}
|
||||
self.api._add_auth_token(headers, download_url, required=False)
|
||||
|
||||
b_collection_path = _download_file(download_url, b_path, artifact_hash, self.api.validate_certs,
|
||||
headers=headers)
|
||||
|
||||
return to_text(b_collection_path, errors='surrogate_or_strict')
|
||||
|
||||
def install(self, path, b_temp_path):
|
||||
if self.skip:
|
||||
display.display("Skipping '%s' as it is already installed" % to_text(self))
|
||||
|
|
@ -189,13 +200,7 @@ class CollectionRequirement:
|
|||
display.display("Installing '%s:%s' to '%s'" % (to_text(self), self.latest_version, collection_path))
|
||||
|
||||
if self.b_path is None:
|
||||
download_url = self._metadata.download_url
|
||||
artifact_hash = self._metadata.artifact_sha256
|
||||
headers = {}
|
||||
self.api._add_auth_token(headers, download_url, required=False)
|
||||
|
||||
self.b_path = _download_file(download_url, b_temp_path, artifact_hash, self.api.validate_certs,
|
||||
headers=headers)
|
||||
self.b_path = self.download(b_temp_path)
|
||||
|
||||
if os.path.exists(b_collection_path):
|
||||
shutil.rmtree(b_collection_path)
|
||||
|
|
@ -493,6 +498,43 @@ def build_collection(collection_path, output_path, force):
|
|||
_build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest)
|
||||
|
||||
|
||||
def download_collections(collections, output_path, apis, validate_certs, no_deps, allow_pre_release):
|
||||
"""
|
||||
Download Ansible collections as their tarball from a Galaxy server to the path specified and creates a requirements
|
||||
file of the downloaded requirements to be used for an install.
|
||||
|
||||
:param collections: The collections to download, should be a list of tuples with (name, requirement, Galaxy Server).
|
||||
:param output_path: The path to download the collections to.
|
||||
:param apis: A list of GalaxyAPIs to query when search for a collection.
|
||||
:param validate_certs: Whether to validate the certificate if downloading a tarball from a non-Galaxy host.
|
||||
:param no_deps: Ignore any collection dependencies and only download the base requirements.
|
||||
:param allow_pre_release: Do not ignore pre-release versions when selecting the latest.
|
||||
"""
|
||||
with _tempdir() as b_temp_path:
|
||||
display.display("Process install dependency map")
|
||||
with _display_progress():
|
||||
dep_map = _build_dependency_map(collections, [], b_temp_path, apis, validate_certs, True, True, no_deps,
|
||||
allow_pre_release=allow_pre_release)
|
||||
|
||||
requirements = []
|
||||
display.display("Starting collection download process to '%s'" % output_path)
|
||||
with _display_progress():
|
||||
for name, requirement in dep_map.items():
|
||||
collection_filename = "%s-%s-%s.tar.gz" % (requirement.namespace, requirement.name,
|
||||
requirement.latest_version)
|
||||
dest_path = os.path.join(output_path, collection_filename)
|
||||
requirements.append({'name': collection_filename, 'version': requirement.latest_version})
|
||||
|
||||
display.display("Downloading collection '%s' to '%s'" % (name, dest_path))
|
||||
b_temp_download_path = requirement.download(b_temp_path)
|
||||
shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict'))
|
||||
|
||||
requirements_path = os.path.join(output_path, 'requirements.yml')
|
||||
display.display("Writing requirements.yml file of downloaded collections to '%s'" % requirements_path)
|
||||
with open(to_bytes(requirements_path, errors='surrogate_or_strict'), mode='wb') as req_fd:
|
||||
req_fd.write(to_bytes(yaml.safe_dump({'collections': requirements}), errors='surrogate_or_strict'))
|
||||
|
||||
|
||||
def publish_collection(collection_path, api, wait, timeout):
|
||||
"""
|
||||
Publish an Ansible collection tarball into an Ansible Galaxy server.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
---
|
||||
- name: create test download dir
|
||||
file:
|
||||
path: '{{ galaxy_dir }}/download'
|
||||
state: directory
|
||||
|
||||
- name: download collection with multiple dependencides
|
||||
command: ansible-galaxy collection download parent_dep.parent_collection -s {{ fallaxy_galaxy_server }}
|
||||
register: download_collection
|
||||
args:
|
||||
chdir: '{{ galaxy_dir }}/download'
|
||||
|
||||
- name: get result of download collection with multiple dependencies
|
||||
find:
|
||||
path: '{{ galaxy_dir }}/download/collections'
|
||||
file_type: file
|
||||
register: download_collection_actual
|
||||
|
||||
- name: assert download collection with multiple dependencies
|
||||
assert:
|
||||
that:
|
||||
- '"Downloading collection ''parent_dep.parent_collection'' to" in download_collection.stdout'
|
||||
- '"Downloading collection ''child_dep.child_collection'' to" in download_collection.stdout'
|
||||
- '"Downloading collection ''child_dep.child_dep2'' to" in download_collection.stdout'
|
||||
- download_collection_actual.examined == 4
|
||||
- download_collection_actual.matched == 4
|
||||
- (download_collection_actual.files[0].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
|
||||
- (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
|
||||
- (download_collection_actual.files[2].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
|
||||
- (download_collection_actual.files[3].path | basename) in ['requirements.yml', 'child_dep-child_dep2-1.2.2.tar.gz', 'child_dep-child_collection-0.9.9.tar.gz', 'parent_dep-parent_collection-1.0.0.tar.gz']
|
||||
|
||||
- name: test install of download requirements file
|
||||
command: ansible-galaxy collection install -r requirements.yml -p '{{ galaxy_dir }}/download'
|
||||
args:
|
||||
chdir: '{{ galaxy_dir }}/download/collections'
|
||||
register: install_download
|
||||
|
||||
- name: get result of test install of download requirements file
|
||||
slurp:
|
||||
path: '{{ galaxy_dir }}/download/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json'
|
||||
register: install_download_actual
|
||||
loop_control:
|
||||
loop_var: collection
|
||||
loop:
|
||||
- namespace: parent_dep
|
||||
name: parent_collection
|
||||
- namespace: child_dep
|
||||
name: child_collection
|
||||
- namespace: child_dep
|
||||
name: child_dep2
|
||||
|
||||
- name: assert test install of download requirements file
|
||||
assert:
|
||||
that:
|
||||
- '"Installing ''parent_dep.parent_collection:1.0.0'' to" in install_download.stdout'
|
||||
- '"Installing ''child_dep.child_collection:0.9.9'' to" in install_download.stdout'
|
||||
- '"Installing ''child_dep.child_dep2:1.2.2'' to" in install_download.stdout'
|
||||
- (install_download_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0'
|
||||
- (install_download_actual.results[1].content | b64decode | from_json).collection_info.version == '0.9.9'
|
||||
- (install_download_actual.results[2].content | b64decode | from_json).collection_info.version == '1.2.2'
|
||||
|
||||
- name: create test requirements file for download
|
||||
copy:
|
||||
content: |
|
||||
collections:
|
||||
- name: namespace1.name1
|
||||
version: 1.1.0-beta.1
|
||||
|
||||
dest: '{{ galaxy_dir }}/download/download.yml'
|
||||
|
||||
- name: download collection with req to custom dir
|
||||
command: ansible-galaxy collection download -r '{{ galaxy_dir }}/download/download.yml' -s {{ fallaxy_ah_server }} -p '{{ galaxy_dir }}/download/collections-custom'
|
||||
register: download_req_custom_path
|
||||
|
||||
- name: get result of download collection with req to custom dir
|
||||
find:
|
||||
path: '{{ galaxy_dir }}/download/collections-custom'
|
||||
file_type: file
|
||||
register: download_req_custom_path_actual
|
||||
|
||||
- name: assert download collection with multiple dependencies
|
||||
assert:
|
||||
that:
|
||||
- '"Downloading collection ''namespace1.name1'' to" in download_req_custom_path.stdout'
|
||||
- download_req_custom_path_actual.examined == 2
|
||||
- download_req_custom_path_actual.matched == 2
|
||||
- (download_req_custom_path_actual.files[0].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz']
|
||||
- (download_req_custom_path_actual.files[1].path | basename) in ['requirements.yml', 'namespace1-name1-1.1.0-beta.1.tar.gz']
|
||||
|
|
@ -31,7 +31,7 @@
|
|||
server: '{{ fallaxy_ah_server }}'
|
||||
|
||||
# We use a module for this so we can speed up the test time.
|
||||
- name: setup test collections for install test
|
||||
- name: setup test collections for install and download test
|
||||
setup_collections:
|
||||
server: '{{ fallaxy_galaxy_server }}'
|
||||
token: '{{ fallaxy_token }}'
|
||||
|
|
@ -148,3 +148,6 @@
|
|||
server: '{{ fallaxy_galaxy_server }}'
|
||||
- name: automation_hub
|
||||
server: '{{ fallaxy_ah_server }}'
|
||||
|
||||
- name: run ansible-galaxy collection download tests
|
||||
include_tasks: download.yml
|
||||
|
|
|
|||
Loading…
Reference in a new issue