diff --git a/.gitignore b/.gitignore index 76b764b9dfa..d0f1ac5f523 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ docs/docsite/*.html docs/docsite/htmlout docs/docsite/rst/cli/ansible-*.rst docs/docsite/rst/cli/ansible.rst +docs/docsite/rst/dev_guide/collections_galaxy_meta.rst docs/docsite/rst/dev_guide/testing/sanity/index.rst.new docs/docsite/rst/modules/*.rst docs/docsite/rst/playbooks_directives.rst diff --git a/docs/docsite/Makefile b/docs/docsite/Makefile index 988dc08f589..cd70faa35d7 100644 --- a/docs/docsite/Makefile +++ b/docs/docsite/Makefile @@ -5,6 +5,7 @@ TESTING_FORMATTER=../bin/testing_formatter.sh KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords CONFIG_DUMPER=../../hacking/build-ansible.py document-config GENERATE_CLI=../../hacking/build-ansible.py generate-man +COLLECTION_DUMPER=../../hacking/build-ansible.py collection-meta ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1) CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}') else @@ -37,7 +38,7 @@ all: docs docs: htmldocs -generate_rst: config cli keywords modules plugins testing +generate_rst: collections_meta config cli keywords modules plugins testing htmldocs: generate_rst CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html @@ -75,9 +76,13 @@ clean: rm -f rst/plugins/*/*.rst rm -f rst/reference_appendices/config.rst rm -f rst/reference_appendices/playbooks_keywords.rst + rm -f rst/dev_guide/collections_galaxy_meta.rst .PHONY: docs clean +collections_meta: ../templates/collections_galaxy_meta.rst.j2 + PYTHONPATH=../../lib $(COLLECTION_DUMPER) --template-file=../templates/collections_galaxy_meta.rst.j2 --output-dir=rst/dev_guide/ ../../lib/ansible/galaxy/data/collections_galaxy_meta.yml + # TODO: make generate_man output dir cli option cli: mkdir -p rst/cli diff --git a/docs/docsite/rst/collections_tech_preview.rst b/docs/docsite/rst/dev_guide/collections_tech_preview.rst similarity index 83% rename from docs/docsite/rst/collections_tech_preview.rst rename to docs/docsite/rst/dev_guide/collections_tech_preview.rst index 005dfb90b27..9578f92139e 100644 --- a/docs/docsite/rst/collections_tech_preview.rst +++ b/docs/docsite/rst/dev_guide/collections_tech_preview.rst @@ -55,50 +55,9 @@ and other tools need in order to package, build and publish the collection.:: galaxy.yml ---------- -This file contains the information about a collection that is necessary for Ansible tools to operate. -``galaxy.yml`` has the following fields (subject to changes and expansion): +A collection must have a ``galaxy.yml`` file that contains the necessary information to build a collection artifact. +See :ref:`collections_galaxy_meta` for details on how this file is structured. -.. code-block:: yaml - - namespace: "namespace_name" - name: "collection_name" - version: "1.0.12" - authors: - - "Author1" - - "Author2 (https://author2.example.com)" - - "Author3 " - dependencies: - "other_namespace.collection1": ">=1.0.0" - "other_namespace.collection2": ">=2.0.0,<3.0.0" - "anderson55.my_collection": "*" # note: "*" selects the highest version available - license: - - "MIT" - tags: - - demo - - collection - repository: "https://www.github.com/my_org/my_collection" - - -Required Fields: - - ``namespace``: the namespace that the collection lives under. It must be a valid Python identifier, - and may only contain alphanumeric characters and underscores. Additionally - the ``namespace`` cannot start with underscores or numbers and cannot contain consecutive - underscores. - - ``name``: the collection's name. Has the same character restrictions as ``namespace``. - - ``version``: the collection's version. To upload to Galaxy, it must be compatible with semantic versioning. - - -Optional Fields: - - ``dependencies``: A dictionary where keys are collections, and values are version - range `specifiers `_. - It is good practice to depend on a version range to minimize conflicts, and pin to a - a major version to protect against breaking changes. For example: ``"user1.collection1": ">=1.2.2,<2.0.0"`` - This field allows other collections as dependencies, not traditional roles. - - ``description``: A short summary description of the collection. - - ``license``: Either a single license or a list of licenses for content inside of a collection. - Galaxy currently only accepts `SPDX `_ licenses. - - ``tags``: a list of tags. These have the same character requirements as ``namespace`` and ``name``. - - ``repository``: URL of originating SCM repository. docs directory --------------- diff --git a/docs/docsite/rst/dev_guide/index.rst b/docs/docsite/rst/dev_guide/index.rst index 9ccc1e87346..411571145bb 100644 --- a/docs/docsite/rst/dev_guide/index.rst +++ b/docs/docsite/rst/dev_guide/index.rst @@ -81,4 +81,6 @@ If you prefer to read the entire guide, here's a list of the pages in order. developing_api developing_rebasing developing_module_utilities + collections_tech_preview + collections_galaxy_meta overview_architecture diff --git a/docs/templates/collections_galaxy_meta.rst.j2 b/docs/templates/collections_galaxy_meta.rst.j2 new file mode 100644 index 00000000000..87c530e0760 --- /dev/null +++ b/docs/templates/collections_galaxy_meta.rst.j2 @@ -0,0 +1,74 @@ +.. _collections_galaxy_meta: + +************************************ +Collection Galaxy Metadata Structure +************************************ + +.. important:: + This feature is available in Ansible 2.8 as a *Technology Preview* and therefore is not fully supported. It should only be used for testing and should not be deployed in a production environment. + Future Galaxy or Ansible releases may introduce breaking changes. + +A key component of an Ansible collection is the ``galaxy.yml`` file placed in the root directory of a collection. This +file contains the metadata of the collection that is used to generate a collection artifact. + +Structure +========= + +The ``galaxy.yml`` file must contain the following keys in valid YAML: + +.. raw:: html + + + {# Header of the documentation -#} + + + + + {% for entry in options %} + + {# key name with required or type label #} + + {# Comments #} + + + {% endfor %} +
KeyComments
+ @{ entry.key }@ +
+ @{ entry.type | documented_type }@ + {% if entry.get('required', False) %} / required{% endif %} +
+ 		+ {% if entry.description is string %} +
@{ entry.description | replace('\n', '\n ') | html_ify }@
+ {% else %} + {% for desc in entry.description %} +
@{ desc | replace('\n', '\n ') | html_ify }@
+ {% endfor %} + {% endif %} +
+
+ +Examples +======== + +.. code-block:: yaml + + namespace: "namespace_name" + name: "collection_name" + version: "1.0.12" + readme: "README.md" + authors: + - "Author1" + - "Author2 (https://author2.example.com)" + - "Author3 " + dependencies: + "other_namespace.collection1": ">=1.0.0" + "other_namespace.collection2": ">=2.0.0,<3.0.0" + "anderson55.my_collection": "*" # note: "*" selects the highest version available + license: + - "MIT" + tags: + - demo + - collection + repository: "https://www.github.com/my_org/my_collection" diff --git a/hacking/build_library/build_ansible/command_plugins/collection_meta.py b/hacking/build_library/build_ansible/command_plugins/collection_meta.py new file mode 100644 index 00000000000..b28797473f2 --- /dev/null +++ b/hacking/build_library/build_ansible/command_plugins/collection_meta.py @@ -0,0 +1,68 @@ +# coding: utf-8 +# Copyright: (c) 2019, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os +import os.path +import pathlib + +import yaml +from jinja2 import Environment, FileSystemLoader +from ansible.module_utils._text import to_bytes + +# Pylint doesn't understand Python3 namespace modules. +from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level +from ..commands import Command # pylint: disable=relative-beyond-top-level +from ..jinja2.filters import documented_type, html_ify # pylint: disable=relative-beyond-top-level + + +DEFAULT_TEMPLATE_FILE = 'collections_galaxy_meta.rst.j2' +DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates' + + +class DocumentCollectionMeta(Command): + name = 'collection-meta' + + @classmethod + def init_parser(cls, add_parser): + parser = add_parser(cls.name, description='Generate collection galaxy.yml documentation from shared metadata') + parser.add_argument("-t", "--template-file", action="store", dest="template_file", + default=DEFAULT_TEMPLATE_FILE, + help="Jinja2 template to use for the config") + parser.add_argument("-T", "--template-dir", action="store", dest="template_dir", + default=DEFAULT_TEMPLATE_DIR, + help="directory containing Jinja2 templates") + parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', + help="Output directory for rst files") + parser.add_argument("collection_defs", metavar="COLLECTION-OPTION-DEFINITIONS.yml", type=str, + help="Source for collection metadata option docs") + + @staticmethod + def main(args): + output_dir = os.path.abspath(args.output_dir) + template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file)) + template_file = os.path.basename(template_file_full_path) + template_dir = os.path.dirname(template_file_full_path) + + with open(args.collection_defs) as f: + options = yaml.safe_load(f) + + env = Environment(loader=FileSystemLoader(template_dir), + variable_start_string="@{", + variable_end_string="}@", + trim_blocks=True) + env.filters['documented_type'] = documented_type + env.filters['html_ify'] = html_ify + + template = env.get_template(template_file) + output_name = os.path.join(output_dir, template_file.replace('.j2', '')) + temp_vars = {'options': options} + + data = to_bytes(template.render(temp_vars)) + update_file_if_different(output_name, data) + + return 0 diff --git a/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py b/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py index 3f61ea62b3b..7407c97f997 100644 --- a/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py +++ b/hacking/build_library/build_ansible/command_plugins/plugin_formatter.py @@ -11,7 +11,6 @@ __metaclass__ = type import datetime import glob import json -import optparse import os import re import sys @@ -34,10 +33,9 @@ except ImportError: import jinja2 import yaml from jinja2 import Environment, FileSystemLoader -from jinja2.runtime import Undefined from ansible.errors import AnsibleError -from ansible.module_utils._text import to_bytes, to_text +from ansible.module_utils._text import to_bytes from ansible.module_utils.common.collections import is_sequence from ansible.module_utils.parsing.convert_bool import boolean from ansible.module_utils.six import iteritems, string_types @@ -48,6 +46,7 @@ from ansible.utils.display import Display # Pylint doesn't understand Python3 namespace modules. from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level from ..commands import Command # pylint: disable=relative-beyond-top-level +from ..jinja2.filters import do_max, documented_type, html_ify, rst_fmt, rst_ify, rst_xline # pylint: disable=relative-beyond-top-level ##################################################################################### @@ -67,14 +66,6 @@ EXAMPLE_YAML = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml' )) -_ITALIC = re.compile(r"I\(([^)]+)\)") -_BOLD = re.compile(r"B\(([^)]+)\)") -_MODULE = re.compile(r"M\(([^)]+)\)") -_URL = re.compile(r"U\(([^)]+)\)") -_LINK = re.compile(r"L\(([^)]+),([^)]+)\)") -_CONST = re.compile(r"C\(([^)]+)\)") -_RULER = re.compile(r"HORIZONTALLINE") - DEPRECATED = b" (D)" pp = PrettyPrinter() @@ -98,74 +89,6 @@ def from_kludge_ns(key): return NS_MAP[key] -# The max filter was added in Jinja2-2.10. Until we can require that version, use this -def do_max(seq): - return max(seq) - - -def rst_ify(text): - ''' convert symbols like I(this is in italics) to valid restructured text ''' - - try: - t = _ITALIC.sub(r"*\1*", text) - t = _BOLD.sub(r"**\1**", t) - t = _MODULE.sub(r":ref:`\1 <\1_module>`", t) - t = _LINK.sub(r"`\1 <\2>`_", t) - t = _URL.sub(r"\1", t) - t = _CONST.sub(r"``\1``", t) - t = _RULER.sub(r"------------", t) - except Exception as e: - raise AnsibleError("Could not process (%s) : %s" % (text, e)) - - return t - - -def html_ify(text): - ''' convert symbols like I(this is in italics) to valid HTML ''' - - if not isinstance(text, string_types): - text = to_text(text) - - t = html_escape(text) - t = _ITALIC.sub(r"\1", t) - t = _BOLD.sub(r"\1", t) - t = _MODULE.sub(r"\1", t) - t = _URL.sub(r"\1", t) - t = _LINK.sub(r"\1", t) - t = _CONST.sub(r"\1", t) - t = _RULER.sub(r"
", t) - - return t.strip() - - -def rst_fmt(text, fmt): - ''' helper for Jinja2 to do format strings ''' - - return fmt % (text) - - -def rst_xline(width, char="="): - ''' return a restructured text line of a given length ''' - - return char * width - - -def documented_type(text): - ''' Convert any python type to a type for documentation ''' - - if isinstance(text, Undefined): - return '-' - if text == 'str': - return 'string' - if text == 'bool': - return 'boolean' - if text == 'int': - return 'integer' - if text == 'dict': - return 'dictionary' - return text - - test_list = partial(is_sequence, include_strings=False) diff --git a/hacking/build_library/build_ansible/jinja2/__init__.py b/hacking/build_library/build_ansible/jinja2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hacking/build_library/build_ansible/jinja2/filters.py b/hacking/build_library/build_ansible/jinja2/filters.py new file mode 100644 index 00000000000..735d07b7ba7 --- /dev/null +++ b/hacking/build_library/build_ansible/jinja2/filters.py @@ -0,0 +1,100 @@ +# Copyright: (c) 2019, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re + +try: + from html import escape as html_escape +except ImportError: + # Python-3.2 or later + import cgi + + def html_escape(text, quote=True): + return cgi.escape(text, quote) + +from jinja2.runtime import Undefined + +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_text +from ansible.module_utils.six import string_types + + +_ITALIC = re.compile(r"I\(([^)]+)\)") +_BOLD = re.compile(r"B\(([^)]+)\)") +_MODULE = re.compile(r"M\(([^)]+)\)") +_URL = re.compile(r"U\(([^)]+)\)") +_LINK = re.compile(r"L\(([^)]+),([^)]+)\)") +_CONST = re.compile(r"C\(([^)]+)\)") +_RULER = re.compile(r"HORIZONTALLINE") + + +def html_ify(text): + ''' convert symbols like I(this is in italics) to valid HTML ''' + + if not isinstance(text, string_types): + text = to_text(text) + + t = html_escape(text) + t = _ITALIC.sub(r"\1", t) + t = _BOLD.sub(r"\1", t) + t = _MODULE.sub(r"\1", t) + t = _URL.sub(r"\1", t) + t = _LINK.sub(r"\1", t) + t = _CONST.sub(r"\1", t) + t = _RULER.sub(r"
", t) + + return t.strip() + + +def documented_type(text): + ''' Convert any python type to a type for documentation ''' + + if isinstance(text, Undefined): + return '-' + if text == 'str': + return 'string' + if text == 'bool': + return 'boolean' + if text == 'int': + return 'integer' + if text == 'dict': + return 'dictionary' + return text + + +# The max filter was added in Jinja2-2.10. Until we can require that version, use this +def do_max(seq): + return max(seq) + + +def rst_ify(text): + ''' convert symbols like I(this is in italics) to valid restructured text ''' + + try: + t = _ITALIC.sub(r"*\1*", text) + t = _BOLD.sub(r"**\1**", t) + t = _MODULE.sub(r":ref:`\1 <\1_module>`", t) + t = _LINK.sub(r"`\1 <\2>`_", t) + t = _URL.sub(r"\1", t) + t = _CONST.sub(r"``\1``", t) + t = _RULER.sub(r"------------", t) + except Exception as e: + raise AnsibleError("Could not process (%s) : %s" % (text, e)) + + return t + + +def rst_fmt(text, fmt): + ''' helper for Jinja2 to do format strings ''' + + return fmt % (text) + + +def rst_xline(width, char="="): + ''' return a restructured text line of a given length ''' + + return char * width diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 99703f8e9a1..b4a495037a7 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -8,17 +8,18 @@ __metaclass__ = type import os.path import re import shutil +import textwrap import time import yaml -from jinja2 import Environment, FileSystemLoader +from jinja2 import BaseLoader, Environment, FileSystemLoader import ansible.constants as C from ansible import context from ansible.cli import CLI from ansible.cli.arguments import option_helpers as opt_help from ansible.errors import AnsibleError, AnsibleOptionsError -from ansible.galaxy import Galaxy +from ansible.galaxy import Galaxy, get_collections_galaxy_meta_info from ansible.galaxy.api import GalaxyAPI from ansible.galaxy.collection import build_collection, install_collections, parse_collections_requirements_file, \ publish_collection @@ -309,6 +310,56 @@ class GalaxyCLI(CLI): raise AnsibleError("Invalid collection name, must be in the format .") + @staticmethod + def _get_skeleton_galaxy_yml(template_path, inject_data): + with open(to_bytes(template_path, errors='surrogate_or_strict'), 'rb') as template_obj: + meta_template = to_text(template_obj.read(), errors='surrogate_or_strict') + + galaxy_meta = get_collections_galaxy_meta_info() + + required_config = [] + optional_config = [] + for meta_entry in galaxy_meta: + config_list = required_config if meta_entry.get('required', False) else optional_config + + value = inject_data.get(meta_entry['key'], None) + if not value: + meta_type = meta_entry.get('type', 'str') + + if meta_type == 'str': + value = '' + elif meta_type == 'list': + value = [] + elif meta_type == 'dict': + value = {} + + meta_entry['value'] = value + config_list.append(meta_entry) + + link_pattern = re.compile(r"L\(([^)]+),\s+([^)]+)\)") + const_pattern = re.compile(r"C\(([^)]+)\)") + + def comment_ify(v): + if isinstance(v, list): + v = ". ".join([l.rstrip('.') for l in v]) + + v = link_pattern.sub(r"\1 <\2>", v) + v = const_pattern.sub(r"'\1'", v) + + return textwrap.fill(v, width=117, initial_indent="# ", subsequent_indent="# ", break_on_hyphens=False) + + def to_yaml(v): + return yaml.safe_dump(v, default_flow_style=False).rstrip() + + env = Environment(loader=BaseLoader) + env.filters['comment_ify'] = comment_ify + env.filters['to_yaml'] = to_yaml + + template = env.from_string(meta_template) + meta_value = template.render({'required_config': required_config, 'optional_config': optional_config}) + + return meta_value + ############################ # execute actions ############################ @@ -359,30 +410,42 @@ class GalaxyCLI(CLI): obj_name = context.CLIARGS['{0}_name'.format(galaxy_type)] inject_data = dict( - author='your name', description='your description', - company='your company (optional)', - license='license (GPL-2.0-or-later, MIT, etc)', - issue_tracker_url='http://example.com/issue/tracker', - repository_url='http://example.com/repository', - documentation_url='http://docs.example.com', - homepage_url='http://example.com', - min_ansible_version=ansible_version[:3], # x.y ansible_plugin_list_dir=get_versioned_doclink('plugins/plugins.html'), ) - if galaxy_type == 'role': - inject_data['role_name'] = obj_name - inject_data['role_type'] = context.CLIARGS['role_type'] - inject_data['license'] = 'license (GPL-2.0-or-later, MIT, etc)' + inject_data.update(dict( + author='your name', + company='your company (optional)', + license='license (GPL-2.0-or-later, MIT, etc)', + role_name=obj_name, + role_type=context.CLIARGS['role_type'], + issue_tracker_url='http://example.com/issue/tracker', + repository_url='http://example.com/repository', + documentation_url='http://docs.example.com', + homepage_url='http://example.com', + min_ansible_version=ansible_version[:3], # x.y + )) + obj_path = os.path.join(init_path, obj_name) elif galaxy_type == 'collection': namespace, collection_name = obj_name.split('.', 1) - inject_data['namespace'] = namespace - inject_data['collection_name'] = collection_name - inject_data['license'] = 'GPL-2.0-or-later' + inject_data.update(dict( + namespace=namespace, + collection_name=collection_name, + version='1.0.0', + readme='README.md', + authors=['your name '], + license=['GPL-2.0-or-later'], + repository='http://example.com/repository', + documentation='http://docs.example.com', + homepage='http://example.com', + issues='http://example.com/issue/tracker', + )) + obj_path = os.path.join(init_path, namespace, collection_name) + b_obj_path = to_bytes(obj_path, errors='surrogate_or_strict') if os.path.exists(b_obj_path): @@ -395,8 +458,10 @@ class GalaxyCLI(CLI): "been modified there already." % to_native(obj_path)) if obj_skeleton is not None: + own_skeleton = False skeleton_ignore_expressions = C.GALAXY_ROLE_SKELETON_IGNORE else: + own_skeleton = True obj_skeleton = self.galaxy.default_role_skeleton_path skeleton_ignore_expressions = ['^.*/.git_keep$'] @@ -428,8 +493,22 @@ class GalaxyCLI(CLI): for f in files: filename, ext = os.path.splitext(f) + if any(r.match(os.path.join(rel_root, f)) for r in skeleton_ignore_re): continue + elif galaxy_type == 'collection' and own_skeleton and rel_root == '.' and f == 'galaxy.yml.j2': + # Special use case for galaxy.yml.j2 in our own default collection skeleton. We build the options + # dynamically which requires special options to be set. + + # The templated data's keys must match the key name but the inject data contains collection_name + # instead of name. We just make a copy and change the key back to name for this file. + template_data = inject_data.copy() + template_data['name'] = template_data.pop('collection_name') + + meta_value = GalaxyCLI._get_skeleton_galaxy_yml(os.path.join(root, rel_root, f), template_data) + b_dest_file = to_bytes(os.path.join(obj_path, rel_root, filename), errors='surrogate_or_strict') + with open(b_dest_file, 'wb') as galaxy_obj: + galaxy_obj.write(to_bytes(meta_value, errors='surrogate_or_strict')) elif ext == ".j2" and not in_templates_dir: src_template = os.path.join(rel_root, f) dest_file = os.path.join(obj_path, rel_root, filename) diff --git a/lib/ansible/galaxy/__init__.py b/lib/ansible/galaxy/__init__.py index 98ae2e93752..dcfdb9049ca 100644 --- a/lib/ansible/galaxy/__init__.py +++ b/lib/ansible/galaxy/__init__.py @@ -24,15 +24,21 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os +import yaml from ansible import context -from ansible.errors import AnsibleError -from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_bytes # default_readme_template # default_meta_template +def get_collections_galaxy_meta_info(): + meta_path = os.path.join(os.path.dirname(__file__), 'data', 'collections_galaxy_meta.yml') + with open(to_bytes(meta_path, errors='surrogate_or_strict'), 'rb') as galaxy_obj: + return yaml.safe_load(galaxy_obj) + + class Galaxy(object): ''' Keeps global galaxy info ''' diff --git a/lib/ansible/galaxy/collection.py b/lib/ansible/galaxy/collection.py index 8d6f73e98bb..df164c1305c 100644 --- a/lib/ansible/galaxy/collection.py +++ b/lib/ansible/galaxy/collection.py @@ -23,6 +23,7 @@ from yaml.error import YAMLError import ansible.constants as C from ansible.errors import AnsibleError +from ansible.galaxy import get_collections_galaxy_meta_info from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils import six from ansible.utils.display import Display @@ -524,11 +525,25 @@ def _tarfile_extract(tar, member): def _get_galaxy_yml(b_galaxy_yml_path): - mandatory_keys = frozenset(['namespace', 'name', 'version', 'authors', 'readme']) - optional_strings = ('description', 'repository', 'documentation', 'homepage', 'issues', 'license_file') - optional_lists = ('license', 'tags', 'authors') # authors isn't optional but this will ensure it is list - optional_dicts = ('dependencies',) - all_keys = frozenset(list(mandatory_keys) + list(optional_strings) + list(optional_lists) + list(optional_dicts)) + meta_info = get_collections_galaxy_meta_info() + + mandatory_keys = set() + string_keys = set() + list_keys = set() + dict_keys = set() + + for info in meta_info: + if info.get('required', False): + mandatory_keys.add(info['key']) + + key_list_type = { + 'str': string_keys, + 'list': list_keys, + 'dict': dict_keys, + }[info.get('type', 'str')] + key_list_type.add(info['key']) + + all_keys = frozenset(list(mandatory_keys) + list(string_keys) + list(list_keys) + list(dict_keys)) try: with open(b_galaxy_yml_path, 'rb') as g_yaml: @@ -549,11 +564,11 @@ def _get_galaxy_yml(b_galaxy_yml_path): % (to_text(b_galaxy_yml_path), ", ".join(extra_keys))) # Add the defaults if they have not been set - for optional_string in optional_strings: + for optional_string in string_keys: if optional_string not in galaxy_yml: galaxy_yml[optional_string] = None - for optional_list in optional_lists: + for optional_list in list_keys: list_val = galaxy_yml.get(optional_list, None) if list_val is None: @@ -561,7 +576,7 @@ def _get_galaxy_yml(b_galaxy_yml_path): elif not isinstance(list_val, list): galaxy_yml[optional_list] = [list_val] - for optional_dict in optional_dicts: + for optional_dict in dict_keys: if optional_dict not in galaxy_yml: galaxy_yml[optional_dict] = {} @@ -655,7 +670,7 @@ def _build_manifest(namespace, name, version, authors, readme, tags, description 'tags': tags, 'description': description, 'license': license_ids, - 'license_file': license_file, + 'license_file': license_file if license_file else None, # Handle galaxy.yml having an empty string (None) 'dependencies': dependencies, 'repository': repository, 'documentation': documentation, diff --git a/lib/ansible/galaxy/data/collections_galaxy_meta.yml b/lib/ansible/galaxy/data/collections_galaxy_meta.yml new file mode 100644 index 00000000000..b7bd69942dc --- /dev/null +++ b/lib/ansible/galaxy/data/collections_galaxy_meta.yml @@ -0,0 +1,98 @@ +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# key: The name of the key as defined in galaxy.yml +# description: Comment/info on the key to be used as the generated doc and auto generated skeleton galaxy.yml file +# required: Whether the key is required (default is no) +# type: The type of value that can be set, aligns to the values in the plugin formatter +--- +- key: namespace + description: + - The namespace of the collection. + - This can be a company/brand/organization or product namespace under which all content lives. + - May only contain alphanumeric characters and underscores. Additionally namespaces cannot start with underscores or + numbers and cannot contain consecutive underscores. + required: yes + type: str + +- key: name + description: + - The name of the collection. + - Has the same character restrictions as C(namespace). + required: yes + type: str + +- key: version + description: + - The version of the collection. + - Must be compatible with semantic versioning. + required: yes + type: str + +- key: readme + description: + - The path to the Markdown (.md) readme file. + - This path is relative to the root of the collection. + required: yes + type: str + +- key: authors + description: + - A list of the collection's content authors. + - Can be just the name or in the format 'Full Name (url) @nicks:irc/im.site#channel'. + required: yes + type: list + +- key: description + description: + - A short summary description of the collection. + type: str + +- key: license + description: + - Either a single license or a list of licenses for content inside of a collection. + - Ansible Galaxy currently only accepts L(SPDX,https://spdx.org/licenses/) licenses + - This key is mutually exclusive with C(license_file). + type: list + +- key: license_file + description: + - The path to the license file for the collection. + - This path is relative to the root of the collection. + - This key is mutually exclusive with C(license). + type: str + +- key: tags + description: + - A list of tags you want to associate with the collection for indexing/searching. + - A tag name has the same character requirements as C(namespace) and C(name). + type: list + +- key: dependencies + description: + - Collections that this collection requires to be installed for it to be usable. + - The key of the dict is the collection label C(namespace.name). + - The value is a version range + L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). + - Multiple version range specifiers can be set and are separated by C(,). + type: dict + +- key: repository + description: + - The URL of the originating SCM repository. + type: str + +- key: documentation + description: + - The URL to any online docs. + type: str + +- key: homepage + description: + - The URL to the homepage of the collection/project. + type: str + +- key: issues + description: + - The URL to the collection issue tracker. + type: str diff --git a/lib/ansible/galaxy/data/default/collection/README.md.j2 b/lib/ansible/galaxy/data/default/collection/README.md.j2 index b28dbc888d9..5e5162206ec 100644 --- a/lib/ansible/galaxy/data/default/collection/README.md.j2 +++ b/lib/ansible/galaxy/data/default/collection/README.md.j2 @@ -1,3 +1,3 @@ # Ansible Collection - {{ namespace }}.{{ collection_name }} -Documentation for the collection. \ No newline at end of file +Documentation for the collection. diff --git a/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2 b/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2 index 0bc8a2c0761..a1c46391b3b 100644 --- a/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2 +++ b/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2 @@ -1,65 +1,11 @@ ### REQUIRED +{% for option in required_config %} +{{ option.description | comment_ify }} +{{ {option.key: option.value} | to_yaml }} +{% endfor %} -# this can be a company/brand/organization or product namespace -# under which all content lives -namespace: {{ namespace }} - - -# the designation of this specific collection -name: {{ collection_name }} - - -# semantic versioning compliant version designation -version: 1.0.0 - -# the filename for the readme file which can be either markdown (.md) -readme: README.md - - -# a list of the collection's content authors -# Ex: 'Full Name (http://site) @nicks:irc/im/site#channel' -authors: -- {{ author }} - - -### OPTIONAL but strongly advised - -# short summary of the collection -description: {{ description }} - - -# Either a single valid SPDX license identifier or a list of valid SPDX license -# identifiers, see https://spdx.org/licenses/. Could also set `license_file` -# instead to point to the file the specifies the license in the collection -# directory. -license: {{ license }} - - -# list of keywords you want to associate the collection -# with for indexing/search systems -tags: [] - - -# A dict of dependencies. A dependency is another collection -# this collection requires to be installed for it to be usable. -# The key of the dict is the collection label (namespace.name) -# and the value is a spec for the semver version required. -dependencies: {} - - -### URLs - -# url of originating SCM repository -repository: {{ repository_url }} - - -# url to online docs -documentation: {{ documentation_url }} - - -# homepage of the collection/project -homepage: {{ homepage_url }} - - -# issue tracker url -issues: {{ issue_tracker_url }} \ No newline at end of file +### OPTIONAL but strongly recommended +{% for option in optional_config %} +{{ option.description | comment_ify }} +{{ {option.key: option.value} | to_yaml }} +{% endfor %} diff --git a/test/sanity/code-smell/no-unwanted-files.py b/test/sanity/code-smell/no-unwanted-files.py index ff736159588..61fbdfdf0b5 100755 --- a/test/sanity/code-smell/no-unwanted-files.py +++ b/test/sanity/code-smell/no-unwanted-files.py @@ -22,6 +22,7 @@ def main(): # allowed special cases 'lib/ansible/config/base.yml', 'lib/ansible/config/module_defaults.yml', + 'lib/ansible/galaxy/data/collections_galaxy_meta.yml', ) skip_directories = ( diff --git a/test/units/cli/test_galaxy.py b/test/units/cli/test_galaxy.py index e2e88ea7bd1..c1ceb422ee0 100644 --- a/test/units/cli/test_galaxy.py +++ b/test/units/cli/test_galaxy.py @@ -496,14 +496,13 @@ def test_collection_default(collection_skeleton): assert metadata['readme'] == 'README.md' assert metadata['version'] == '1.0.0' assert metadata['description'] == 'your description' - assert metadata['license'] == 'GPL-2.0-or-later' + assert metadata['license'] == ['GPL-2.0-or-later'] assert metadata['tags'] == [] assert metadata['dependencies'] == {} assert metadata['documentation'] == 'http://docs.example.com' assert metadata['repository'] == 'http://example.com/repository' assert metadata['homepage'] == 'http://example.com' assert metadata['issues'] == 'http://example.com/issue/tracker' - assert len(metadata) == 13 for d in ['docs', 'plugins', 'roles']: assert os.path.isdir(os.path.join(collection_skeleton, d)), \ diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py index 2eca2473aa7..02441baa185 100644 --- a/test/units/galaxy/test_collection.py +++ b/test/units/galaxy/test_collection.py @@ -215,11 +215,6 @@ readme: README.md"""], indirect=True) def test_defaults_galaxy_yml(galaxy_yml): actual = collection._get_galaxy_yml(galaxy_yml) - assert sorted(list(actual.keys())) == [ - 'authors', 'dependencies', 'description', 'documentation', 'homepage', 'issues', 'license_file', 'license_ids', - 'name', 'namespace', 'readme', 'repository', 'tags', 'version', - ] - assert actual['namespace'] == 'namespace' assert actual['name'] == 'collection' assert actual['authors'] == ['Jordan']