From bd18be6c0c3cb45b0c67f983883a0acf246315aa Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Thu, 4 Feb 2021 13:42:45 -0500 Subject: [PATCH] Fix ansible-galaxy collection subdir searching and update documentation (#73406) * Ensure there is a single source of collection metadata * Allow collection subdirs to be detected by a galaxy.yml or MANIFEST.json * Add documentation about installing and downloading collection directories * Add an example for downloading a git repository * Update documented valid metadata sources for installing git repositories Co-authored-by: Sviatoslav Sydorenko Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com> --- .../installing_collections.txt | 23 +++++++++++++++ .../installing_collections_git_repo.txt | 10 +++---- .../rst/user_guide/collections_using.rst | 24 ++++++++++++++++ .../dependency_resolution/dataclasses.py | 28 +++++++++++++++---- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/docs/docsite/rst/shared_snippets/installing_collections.txt b/docs/docsite/rst/shared_snippets/installing_collections.txt index 34486f7e362..686e73fc167 100644 --- a/docs/docsite/rst/shared_snippets/installing_collections.txt +++ b/docs/docsite/rst/shared_snippets/installing_collections.txt @@ -24,6 +24,29 @@ You can also directly use the tarball from your build: ansible-galaxy collection install my_namespace-my_collection-1.0.0.tar.gz -p ./collections +You can build and install a collection from a local source directory. The ``ansible-galaxy`` utility builds the collection using the ``MANIFEST.json`` or ``galaxy.yml`` +metadata in the directory. + +.. code-block:: bash + + ansible-galaxy collection install /path/to/collection -p ./collections + +You can also install multiple collections in a namespace directory. + +.. code-block:: text + + ns/ + ├── collection1/ + │   ├── MANIFEST.json + │   └── plugins/ + └── collection2/ + ├── galaxy.yml + └── plugins/ + +.. code-block:: bash + + ansible-galaxy collection install /path/to/ns -p ./collections + .. note:: The install command automatically appends the path ``ansible_collections`` to the one specified with the ``-p`` option unless the parent directory is already in a folder called ``ansible_collections``. diff --git a/docs/docsite/rst/shared_snippets/installing_collections_git_repo.txt b/docs/docsite/rst/shared_snippets/installing_collections_git_repo.txt index 7eb87829a5b..b6b0384a715 100644 --- a/docs/docsite/rst/shared_snippets/installing_collections_git_repo.txt +++ b/docs/docsite/rst/shared_snippets/installing_collections_git_repo.txt @@ -1,4 +1,4 @@ -You can install a collection in a git repository by providing the URI to the repository instead of a collection name or path to a ``tar.gz`` file. The collection must contain a ``galaxy.yml`` file, which will be used to generate the would-be collection artifact data from the directory. The URI should be prefixed with ``git+`` (or with ``git@`` to use a private repository with ssh authentication) and optionally supports a comma-separated `git commit-ish `_ version (for example, a commit or tag). +You can install a collection in a git repository by providing the URI to the repository instead of a collection name or path to a ``tar.gz`` file. The collection must contain a ``galaxy.yml`` or ``MANIFEST.json`` file, which will be used to generate the would-be collection artifact data from the directory. The URI should be prefixed with ``git+`` (or with ``git@`` to use a private repository with ssh authentication) and optionally supports a comma-separated `git commit-ish `_ version (for example, a commit or tag). .. warning:: @@ -35,7 +35,7 @@ Default repository search locations There are two paths searched in a repository for collections by default. -The first is the ``galaxy.yml`` file in the top level of the repository path. If the ``galaxy.yml`` file exists it's used as the collection metadata and the individual collection will be installed. +The first is the ``galaxy.yml`` or ``MANIFEST.json`` file in the top level of the repository path. If the file exists it's used as the collection metadata and the individual collection will be installed. .. code-block:: text @@ -46,13 +46,13 @@ The first is the ``galaxy.yml`` file in the top level of the repository path. If │   └── module_utils/ └─── README.md -The second is a ``galaxy.yml`` file in each directory in the repository path (one level deep). In this scenario, each directory with a ``galaxy.yml`` is installed as a collection. +The second is a ``galaxy.yml`` or ``MANIFEST.json`` file in each directory in the repository path (one level deep). In this scenario, each directory with a metadata file is installed as a collection. .. code-block:: text directory/ ├── docs/ - ├── galaxy.yml + ├── MANIFEST.json ├── plugins/ │   ├── inventory/ │   └── modules/ @@ -61,7 +61,7 @@ The second is a ``galaxy.yml`` file in each directory in the repository path (on Specifying the location to search for collections ------------------------------------------------- -If you have a different repository structure or only want to install a subset of collections, you can add a fragment to the end of your URI (before the optional comma-separated version) to indicate which path ansible-galaxy should inspect for ``galaxy.yml`` file(s). The path should be a directory to a collection or multiple collections (rather than the path to a ``galaxy.yml`` file). +If you have a different repository structure or only want to install a subset of collections, you can add a fragment to the end of your URI (before the optional comma-separated version) to indicate which path ansible-galaxy should inspect for metadata file(s). The path should be a directory to a collection or multiple collections (rather than the path to a ``galaxy.yml`` file or ``MANIFEST.json`` file). .. code-block:: text diff --git a/docs/docsite/rst/user_guide/collections_using.rst b/docs/docsite/rst/user_guide/collections_using.rst index edab2aaa89f..0f1bb65a3d9 100644 --- a/docs/docsite/rst/user_guide/collections_using.rst +++ b/docs/docsite/rst/user_guide/collections_using.rst @@ -97,6 +97,30 @@ requirements file in the format documented with :ref:`collection_requirements_fi ansible-galaxy collection download -r requirements.yml +You can also download a source collection directory. The collection is built with the mandatory ``galaxy.yml`` file. + +.. code-block:: bash + + ansible-galaxy collection download /path/to/collection + + ansible-galaxy collection download git+file:///path/to/collection/.git + +You can download multiple source collections from a single namespace by providing the path to the namespace. + +.. code-block:: text + + ns/ + ├── collection1/ + │   ├── galaxy.yml + │   └── plugins/ + └── collection2/ + ├── galaxy.yml + └── plugins/ + +.. code-block:: bash + + ansible-galaxy collection install /path/to/ns + All the collections are downloaded by default to the ``./collections`` folder but you can use ``-p`` or ``--download-path`` to specify another path: diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py index bea5dacc960..002578d96ca 100644 --- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py +++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py @@ -81,17 +81,19 @@ def _is_collection_dir(dir_path): def _find_collections_in_subdirs(dir_path): b_dir_path = to_bytes(dir_path, errors='surrogate_or_strict') - galaxy_yml_glob_pattern = os.path.join( + + subdir_glob_pattern = os.path.join( b_dir_path, # b'*', # namespace is supposed to be top-level per spec b'*', # collection name - _GALAXY_YAML, - ) - return ( - os.path.dirname(galaxy_yml) - for galaxy_yml in iglob(galaxy_yml_glob_pattern) ) + for subdir in iglob(subdir_glob_pattern): + if os.path.isfile(os.path.join(subdir, _MANIFEST_JSON)): + yield subdir + elif os.path.isfile(os.path.join(subdir, _GALAXY_YAML)): + yield subdir + def _is_collection_namespace_dir(tested_str): return any(_find_collections_in_subdirs(tested_str)) @@ -280,6 +282,20 @@ class _ComputedReqKindsMixin: req_type = 'file' req_source = src_path elif _is_collection_dir(src_path): + if _is_installed_collection_dir(src_path) and _is_collection_src_dir(src_path): + # Note that ``download`` requires a dir with a ``galaxy.yml`` and fails if it + # doesn't exist, but if a ``MANIFEST.json`` also exists, it would be used + # instead of the ``galaxy.yml``. + raise AnsibleError( + u"Collection requirement at '{path!s}' has both a {manifest_json!s} " + u"file and a {galaxy_yml!s}.\nThe requirement must either be an installed " + u"collection directory or a source collection directory, not both.". + format( + path=to_text(src_path, errors='surrogate_or_strict'), + manifest_json=to_text(_MANIFEST_JSON), + galaxy_yml=to_text(_GALAXY_YAML), + ) + ) req_type = 'dir' req_source = src_path elif _is_collection_namespace_dir(src_path):