Add the ability to ignore files and collection build (#64688)

This commit is contained in:
Jordan Borean 2019-11-14 05:02:58 +10:00 committed by GitHub
parent bf19060683
commit f8f7662850
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 159 additions and 28 deletions

View file

@ -0,0 +1,3 @@
minor_changes:
- ansible-galaxy - Always ignore the ``tests/output`` directory when building a collection as it is used by ``ansible-test`` for test output (https://github.com/ansible/ansible/issues/59228).
- ansible-galaxy - Added the ability to ignore further files and folders using a pattern with the ``build_ignore`` key in a collection's ``galaxy.yml`` (https://github.com/ansible/ansible/issues/59228).

File diff suppressed because one or more lines are too long

View file

@ -234,15 +234,50 @@ a tarball of the built collection in the current directory which can be uploaded
.. note:: .. note::
* Certain files and folders are excluded when building the collection artifact. This is not currently configurable and is a work in progress so the collection artifact may contain files you would not wish to distribute. * Certain files and folders are excluded when building the collection artifact. See :ref:`ignoring_files_and_folders_collections` to exclude other files you would not wish to distribute.
* If you used the now-deprecated ``Mazer`` tool for any of your collections, delete any and all files it added to your :file:`releases/` directory before you build your collection with ``ansible-galaxy``. * If you used the now-deprecated ``Mazer`` tool for any of your collections, delete any and all files it added to your :file:`releases/` directory before you build your collection with ``ansible-galaxy``.
* You must also delete the :file:`tests/output` directory if you have been testing with ``ansible-test``.
* The current Galaxy maximum tarball size is 2 MB. * The current Galaxy maximum tarball size is 2 MB.
This tarball is mainly intended to upload to Galaxy This tarball is mainly intended to upload to Galaxy
as a distribution method, but you can use it directly to install the collection on target systems. as a distribution method, but you can use it directly to install the collection on target systems.
.. _ignoring_files_and_folders_collections:
Ignoring files and folders
^^^^^^^^^^^^^^^^^^^^^^^^^^
By default the build step will include all the files in the collection directory in the final build artifact except for the following:
* ``galaxy.yml``
* ``*.pyc``
* ``*.retry``
* ``tests/output``
* previously built artifacts in the root directory
* Various version control directories like ``.git/``
To exclude other files and folders when building the collection, you can set a list of file glob-like patterns in the
``build_ignore`` key in the collection's ``galaxy.yml`` file. These patterns use the following special characters for
wildcard matching:
* ``*``: Matches everything
* ``?``: Matches any single character
* ``[seq]``: Matches and character in seq
* ``[!seq]``:Matches any character not in seq
For example, if you wanted to exclude the :file:`sensitive` folder within the ``playbooks`` folder as well any ``.tar.gz`` archives you
can set the following in your ``galaxy.yml`` file:
.. code-block:: yaml
build_ignore:
- playbooks/sensitive
- '*.tar.gz'
.. note::
This feature is only supported when running ``ansible-galaxy collection build`` with Ansible 2.10 or newer.
.. _trying_collection_locally: .. _trying_collection_locally:
Trying collection locally Trying collection locally

View file

@ -43,6 +43,15 @@ The ``galaxy.yml`` file must contain the following keys in valid YAML:
required required
{%- endif %} {%- endif %}
{% if 'version_added' in entry -%}
.. rst-class:: value-added-in
|br| version_added: @{ entry.version_added }@
|_|
{%- endif %}
- {% for desc in entry.description -%} - {% for desc in entry.description -%}
@{ desc | trim | rst_ify }@ @{ desc | trim | rst_ify }@

View file

@ -651,6 +651,7 @@ class GalaxyCLI(CLI):
documentation='http://docs.example.com', documentation='http://docs.example.com',
homepage='http://example.com', homepage='http://example.com',
issues='http://example.com/issue/tracker', issues='http://example.com/issue/tracker',
build_ignore=[],
)) ))
obj_path = os.path.join(init_path, namespace, collection_name) obj_path = os.path.join(init_path, namespace, collection_name)

View file

@ -353,7 +353,8 @@ def build_collection(collection_path, output_path, force):
raise AnsibleError("The collection galaxy.yml path '%s' does not exist." % to_native(b_galaxy_path)) raise AnsibleError("The collection galaxy.yml path '%s' does not exist." % to_native(b_galaxy_path))
collection_meta = _get_galaxy_yml(b_galaxy_path) collection_meta = _get_galaxy_yml(b_galaxy_path)
file_manifest = _build_files_manifest(b_collection_path, collection_meta['namespace'], collection_meta['name']) file_manifest = _build_files_manifest(b_collection_path, collection_meta['namespace'], collection_meta['name'],
collection_meta['build_ignore'])
collection_manifest = _build_manifest(**collection_meta) collection_manifest = _build_manifest(**collection_meta)
collection_output = os.path.join(output_path, "%s-%s-%s.tar.gz" % (collection_meta['namespace'], collection_output = os.path.join(output_path, "%s-%s-%s.tar.gz" % (collection_meta['namespace'],
@ -598,12 +599,18 @@ def _get_galaxy_yml(b_galaxy_yml_path):
return galaxy_yml return galaxy_yml
def _build_files_manifest(b_collection_path, namespace, name): def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
# Contains tuple of (b_filename, only root) where 'only root' means to only ignore the file in the root dir # We always ignore .pyc and .retry files as well as some well known version control directories. The ignore
b_ignore_files = frozenset([(b'*.pyc', False), (b'*.retry', False), # patterns can be extended by the build_ignore key in galaxy.yml
(to_bytes('{0}-{1}-*.tar.gz'.format(namespace, name)), True)]) b_ignore_patterns = [
b_ignore_dirs = frozenset([(b'CVS', False), (b'.bzr', False), (b'.hg', False), (b'.git', False), (b'.svn', False), b'galaxy.yml',
(b'__pycache__', False), (b'.tox', False)]) b'*.pyc',
b'*.retry',
b'tests/output', # Ignore ansible-test result output directory.
to_bytes('{0}-{1}-*.tar.gz'.format(namespace, name)), # Ignores previously built artifacts in the root dir.
]
b_ignore_patterns += [to_bytes(p) for p in ignore_patterns]
b_ignore_dirs = frozenset([b'CVS', b'.bzr', b'.hg', b'.git', b'.svn', b'__pycache__', b'.tox'])
entry_template = { entry_template = {
'name': None, 'name': None,
@ -626,16 +633,15 @@ def _build_files_manifest(b_collection_path, namespace, name):
} }
def _walk(b_path, b_top_level_dir): def _walk(b_path, b_top_level_dir):
is_root = b_path == b_top_level_dir
for b_item in os.listdir(b_path): for b_item in os.listdir(b_path):
b_abs_path = os.path.join(b_path, b_item) b_abs_path = os.path.join(b_path, b_item)
b_rel_base_dir = b'' if b_path == b_top_level_dir else b_path[len(b_top_level_dir) + 1:] b_rel_base_dir = b'' if b_path == b_top_level_dir else b_path[len(b_top_level_dir) + 1:]
rel_path = to_text(os.path.join(b_rel_base_dir, b_item), errors='surrogate_or_strict') b_rel_path = os.path.join(b_rel_base_dir, b_item)
rel_path = to_text(b_rel_path, errors='surrogate_or_strict')
if os.path.isdir(b_abs_path): if os.path.isdir(b_abs_path):
if any(b_item == b_path for b_path, root_only in b_ignore_dirs if any(b_item == b_path for b_path in b_ignore_dirs) or \
if not root_only or root_only == is_root): any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path)) display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
continue continue
@ -655,10 +661,7 @@ def _build_files_manifest(b_collection_path, namespace, name):
_walk(b_abs_path, b_top_level_dir) _walk(b_abs_path, b_top_level_dir)
else: else:
if b_item == b'galaxy.yml': if any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
continue
elif any(fnmatch.fnmatch(b_item, b_pattern) for b_pattern, root_only in b_ignore_files
if not root_only or root_only == is_root):
display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path)) display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
continue continue

View file

@ -96,3 +96,15 @@
description: description:
- The URL to the collection issue tracker. - The URL to the collection issue tracker.
type: str type: str
- key: build_ignore
description:
- A list of file glob-like patterns used to filter any files or directories
that should not be included in the build artifact.
- A pattern is matched from the relative path of the file or directory of the
collection directory.
- This uses C(fnmatch) to match the files or directories.
- Some directories and files like C(galaxy.yml), C(*.pyc), C(*.retry), and
C(.git) are always filtered.
type: list
version_added: '2.10'

View file

@ -248,24 +248,37 @@ def test_build_ignore_files_and_folders(collection_input, monkeypatch):
git_folder = os.path.join(input_dir, '.git') git_folder = os.path.join(input_dir, '.git')
retry_file = os.path.join(input_dir, 'ansible.retry') retry_file = os.path.join(input_dir, 'ansible.retry')
tests_folder = os.path.join(input_dir, 'tests', 'output')
tests_output_file = os.path.join(tests_folder, 'result.txt')
os.makedirs(git_folder) os.makedirs(git_folder)
os.makedirs(tests_folder)
with open(retry_file, 'w+') as ignore_file: with open(retry_file, 'w+') as ignore_file:
ignore_file.write('random') ignore_file.write('random')
ignore_file.flush() ignore_file.flush()
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection') with open(tests_output_file, 'w+') as tests_file:
tests_file.write('random')
tests_file.flush()
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
assert actual['format'] == 1 assert actual['format'] == 1
for manifest_entry in actual['files']: for manifest_entry in actual['files']:
assert manifest_entry['name'] not in ['.git', 'ansible.retry', 'galaxy.yml'] assert manifest_entry['name'] not in ['.git', 'ansible.retry', 'galaxy.yml', 'tests/output', 'tests/output/result.txt']
expected_msgs = [ expected_msgs = [
"Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
"Skipping '%s' for collection build" % to_text(retry_file), "Skipping '%s' for collection build" % to_text(retry_file),
"Skipping '%s' for collection build" % to_text(git_folder), "Skipping '%s' for collection build" % to_text(git_folder),
"Skipping '%s' for collection build" % to_text(tests_folder),
] ]
assert mock_display.call_count == 2 assert mock_display.call_count == 4
assert mock_display.mock_calls[0][1][0] in expected_msgs assert mock_display.mock_calls[0][1][0] in expected_msgs
assert mock_display.mock_calls[1][1][0] in expected_msgs assert mock_display.mock_calls[1][1][0] in expected_msgs
assert mock_display.mock_calls[2][1][0] in expected_msgs
assert mock_display.mock_calls[3][1][0] in expected_msgs
def test_build_ignore_older_release_in_root(collection_input, monkeypatch): def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
@ -285,7 +298,7 @@ def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
file_obj.write('random') file_obj.write('random')
file_obj.flush() file_obj.flush()
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection') actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
assert actual['format'] == 1 assert actual['format'] == 1
plugin_release_found = False plugin_release_found = False
@ -296,8 +309,62 @@ def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
assert plugin_release_found assert plugin_release_found
assert mock_display.call_count == 1 expected_msgs = [
assert mock_display.mock_calls[0][1][0] == "Skipping '%s' for collection build" % to_text(release_file) "Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
"Skipping '%s' for collection build" % to_text(release_file)
]
assert mock_display.call_count == 2
assert mock_display.mock_calls[0][1][0] in expected_msgs
assert mock_display.mock_calls[1][1][0] in expected_msgs
def test_build_ignore_patterns(collection_input, monkeypatch):
input_dir = collection_input[0]
mock_display = MagicMock()
monkeypatch.setattr(Display, 'vvv', mock_display)
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection',
['*.md', 'plugins/action', 'playbooks/*.j2'])
assert actual['format'] == 1
expected_missing = [
'README.md',
'docs/My Collection.md',
'plugins/action',
'playbooks/templates/test.conf.j2',
'playbooks/templates/subfolder/test.conf.j2',
]
# Files or dirs that are close to a match but are not, make sure they are present
expected_present = [
'docs',
'roles/common/templates/test.conf.j2',
'roles/common/templates/subfolder/test.conf.j2',
]
actual_files = [e['name'] for e in actual['files']]
for m in expected_missing:
assert m not in actual_files
for p in expected_present:
assert p in actual_files
expected_msgs = [
"Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
"Skipping '%s/README.md' for collection build" % to_text(input_dir),
"Skipping '%s/docs/My Collection.md' for collection build" % to_text(input_dir),
"Skipping '%s/plugins/action' for collection build" % to_text(input_dir),
"Skipping '%s/playbooks/templates/test.conf.j2' for collection build" % to_text(input_dir),
"Skipping '%s/playbooks/templates/subfolder/test.conf.j2' for collection build" % to_text(input_dir),
]
assert mock_display.call_count == len(expected_msgs)
assert mock_display.mock_calls[0][1][0] in expected_msgs
assert mock_display.mock_calls[1][1][0] in expected_msgs
assert mock_display.mock_calls[2][1][0] in expected_msgs
assert mock_display.mock_calls[3][1][0] in expected_msgs
assert mock_display.mock_calls[4][1][0] in expected_msgs
assert mock_display.mock_calls[5][1][0] in expected_msgs
def test_build_ignore_symlink_target_outside_collection(collection_input, monkeypatch): def test_build_ignore_symlink_target_outside_collection(collection_input, monkeypatch):
@ -309,7 +376,7 @@ def test_build_ignore_symlink_target_outside_collection(collection_input, monkey
link_path = os.path.join(input_dir, 'plugins', 'connection') link_path = os.path.join(input_dir, 'plugins', 'connection')
os.symlink(outside_dir, link_path) os.symlink(outside_dir, link_path)
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection') actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
for manifest_entry in actual['files']: for manifest_entry in actual['files']:
assert manifest_entry['name'] != 'plugins/connection' assert manifest_entry['name'] != 'plugins/connection'
@ -333,7 +400,7 @@ def test_build_copy_symlink_target_inside_collection(collection_input):
os.symlink(roles_target, roles_link) os.symlink(roles_target, roles_link)
actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection') actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
linked_entries = [e for e in actual['files'] if e['name'].startswith('playbooks/roles/linked')] linked_entries = [e for e in actual['files'] if e['name'].startswith('playbooks/roles/linked')]
assert len(linked_entries) == 3 assert len(linked_entries) == 3