From af2073d057b30d57d1d4e5e476b25c5c61e14f47 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Sat, 5 Aug 2017 11:28:21 -0700 Subject: [PATCH] metadata 1.1 * Add network value to support_by field. * New support_by value, certified * Deprecate curated in favor of certified * Add conversion from 1.0 to 1.1 to metadata-tool * Add supported by Red Hat field to ansible-doc output --- docs/bin/plugin_formatter.py | 3 - .../developing_modules_checklist.rst | 2 +- .../developing_modules_documenting.rst | 30 +++++++--- .../dev_guide/developing_modules_general.rst | 4 +- .../developing_modules_general_OLD.rst | 22 +++---- docs/docsite/rst/modules_support.rst | 14 +++-- docs/templates/plugin.rst.j2 | 8 +-- hacking/metadata-tool.py | 57 ++++++++++++++----- lib/ansible/cli/doc.py | 39 +++++++++++++ lib/ansible/parsing/metadata.py | 5 ++ .../module_precedence/lib_no_extension/ping | 2 +- .../lib_with_extension/ping.py | 2 +- .../multiple_roles/bar/library/ping.py | 2 +- .../multiple_roles/foo/library/ping.py | 2 +- .../roles_no_extension/foo/library/ping | 2 +- .../roles_with_extension/foo/library/ping.py | 2 +- test/sanity/validate-modules/main.py | 4 +- test/sanity/validate-modules/schema.py | 16 +++++- test/units/parsing/test_metadata.py | 30 +++++----- 19 files changed, 175 insertions(+), 71 deletions(-) diff --git a/docs/bin/plugin_formatter.py b/docs/bin/plugin_formatter.py index 4d2add33ee2..b858046b865 100755 --- a/docs/bin/plugin_formatter.py +++ b/docs/bin/plugin_formatter.py @@ -255,9 +255,6 @@ def process_module(module, options, env, template, outputname, module_map, alias if doc is None: sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) - if metadata is None: - sys.exit("*** ERROR: MODULE MISSING METADATA: %s, %s ***\n" % (fname, module)) - if deprecated and 'deprecated' not in doc: sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) diff --git a/docs/docsite/rst/dev_guide/developing_modules_checklist.rst b/docs/docsite/rst/dev_guide/developing_modules_checklist.rst index 61230d114b6..c936b07dab5 100644 --- a/docs/docsite/rst/dev_guide/developing_modules_checklist.rst +++ b/docs/docsite/rst/dev_guide/developing_modules_checklist.rst @@ -31,7 +31,7 @@ The following checklist items are important guidelines for people who want to c ANSIBLE_METADATA = {'status': ['preview'], 'supported_by': 'community', - 'metadata_version': '1.0'} + 'metadata_version': '1.1'} The complete module metadata specification is here: `Ansible metadata block `_ diff --git a/docs/docsite/rst/dev_guide/developing_modules_documenting.rst b/docs/docsite/rst/dev_guide/developing_modules_documenting.rst index 42a5765c83a..fb494a1887c 100644 --- a/docs/docsite/rst/dev_guide/developing_modules_documenting.rst +++ b/docs/docsite/rst/dev_guide/developing_modules_documenting.rst @@ -75,7 +75,7 @@ For new modules, the following block can be simply added into your module .. code-block:: python - ANSIBLE_METADATA = {'metadata_version': '1.0', + ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['preview'], 'supported_by': 'community'} @@ -92,7 +92,7 @@ For new modules, the following block can be simply added into your module ANSIBLE_METADATA doesn't look quite right because of this. Module metadata should be fixed before checking it into the repository. -Version 1.0 of the metadata +Version 1.1 of the metadata +++++++++++++++++++++++++++ Structure @@ -101,7 +101,7 @@ Structure .. code-block:: python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'supported_by': 'community', 'status': ['preview', 'deprecated'] } @@ -115,13 +115,17 @@ Fields of the metadata. We’ll increment Y if we add fields or legal values to an existing field. We’ll increment X if we remove fields or values or change the type or meaning of a field. + Current metadata_version is "1.1" :supported_by: This field records who supports the module. Default value is ``community``. Values are: - :core: - :curated: - :community: - + * core + * network + * certfied + * community + * curated (Deprecated. Modules in this category should probably be core or + certified instead) + For information on what the support level values entail, please see `Modules Support `_. @@ -142,6 +146,18 @@ Fields kept so that documentation can be built. The documentation helps users port from the removed module to new modules. +Changes from Version 1.0 +++++++++++++++++++++++++ + +:metadata_version: Version updated from 1.0 to 1.1 +:supported_by: All substantive changes were to potential values of the supported_by field + + * Added the certified value + * Deprecated the curated value, modules shipped with Ansible will use + certified instead. Third party modules are encouraged not to use this as + it is meaningless within Ansible proper. + * Added the network value + DOCUMENTATION Block ------------------- diff --git a/docs/docsite/rst/dev_guide/developing_modules_general.rst b/docs/docsite/rst/dev_guide/developing_modules_general.rst index 8e33b69a6e8..1ddb6faee1c 100644 --- a/docs/docsite/rst/dev_guide/developing_modules_general.rst +++ b/docs/docsite/rst/dev_guide/developing_modules_general.rst @@ -56,9 +56,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' diff --git a/docs/docsite/rst/dev_guide/developing_modules_general_OLD.rst b/docs/docsite/rst/dev_guide/developing_modules_general_OLD.rst index 203c5c45754..4e046af435b 100644 --- a/docs/docsite/rst/dev_guide/developing_modules_general_OLD.rst +++ b/docs/docsite/rst/dev_guide/developing_modules_general_OLD.rst @@ -88,9 +88,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' @@ -401,9 +401,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' @@ -714,9 +714,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' @@ -1027,9 +1027,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' @@ -1340,9 +1340,9 @@ working on a whole new file. Here is an example: #!/usr/bin/python ANSIBLE_METADATA = { - 'metadata_version': '1.0', + 'metadata_version': '1.1', 'status': ['preview'], - 'supported_by': 'curated' + 'supported_by': 'community' } DOCUMENTATION = ''' @@ -1569,4 +1569,4 @@ Credit ====== A *huge* thank you to the Ansible team at Red Hat for providing not only -a great product but also the willingness to help out contributors! \ No newline at end of file +a great product but also the willingness to help out contributors! diff --git a/docs/docsite/rst/modules_support.rst b/docs/docsite/rst/modules_support.rst index 315952289d7..951098edd72 100644 --- a/docs/docsite/rst/modules_support.rst +++ b/docs/docsite/rst/modules_support.rst @@ -18,15 +18,21 @@ The modules are hosted on GitHub in a subdirectory of the `ansible XXX. We don't yet have anything beyond 1.0 + metadata = convert_metadata_1_0_to_1_1(metadata) + + if metadata['metadata_version'] == '1.1' and StrictVersion('1.1') < requested_version: + # 1.1 version => XXX. We don't yet have anything beyond 1.1 # so there's nothing here pass @@ -469,8 +494,10 @@ def report(version=None): print('== Supported by core ==') pprint(sorted(support['core'])) - print('== Supported by value curated ==') - pprint(sorted(support['curated'])) + print('== Supported by value certified ==') + pprint(sorted(support['certified'])) + print('== Supported by value network ==') + pprint(sorted(support['network'])) print('== Supported by community ==') pprint(sorted(support['community'])) print('') @@ -487,8 +514,8 @@ def report(version=None): print('== Summary ==') print('No Metadata: {0} Has Metadata: {1}'.format(len(no_metadata), len(has_metadata))) - print('Supported by core: {0} Supported by community: {1} Supported by value curated: {2}'.format(len(support['core']), - len(support['community']), len(support['curated']))) + print('Support level: core: {0} community: {1} certified: {2} network: {3}'.format(len(support['core']), + len(support['community']), len(support['certified']), len(support['network']))) print('Status StableInterface: {0} Status Preview: {1} Status Deprecated: {2} Status Removed: {3}'.format(len(status['stableinterface']), len(status['preview']), len(status['deprecated']), len(status['removed']))) diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 664d2b72857..a2a7df87769 100644 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -361,6 +361,36 @@ class DocCLI(CLI): text.append(self._dump_yaml({k: opt[k]}, opt_indent)) text.append('') + @staticmethod + def get_support_block(doc): + # Note: 'curated' is deprecated and not used in any of the modules we ship + support_level_msg = {'core': 'The Ansible Core Team', + 'network': 'The Ansible Network Team', + 'certified': 'an Ansible Partner', + 'community': 'The Ansible Community', + 'curated': 'A Third Party', + } + if doc['metadata'].get('metadata_version') in ('1.0', '1.1'): + return [" * This module is maintained by %s" % support_level_msg[doc['metadata']['supported_by']]] + + return [] + + @staticmethod + def get_metadata_block(doc): + text = [] + if doc['metadata'].get('metadata_version') in ('1.0', '1.1'): + text.append("METADATA:") + text.append('\tSUPPORT LEVEL: %s' % doc['metadata']['supported_by']) + + for k in (m for m in doc['metadata'] if m not in ('version', 'metadata_version', 'supported_by')): + if isinstance(k, list): + text.append("\t%s: %s" % (k.capitalize(), ", ".join(doc['metadata'][k]))) + else: + text.append("\t%s: %s" % (k.capitalize(), doc['metadata'][k])) + return text + + return [] + def get_man_text(self, doc): IGNORE = frozenset(['module', 'docuri', 'version_added', 'short_description', 'now_date']) @@ -381,6 +411,10 @@ class DocCLI(CLI): if 'deprecated' in doc and doc['deprecated'] is not None and len(doc['deprecated']) > 0: text.append("DEPRECATED: \n%s\n" % doc.pop('deprecated')) + support_block = self.get_support_block(doc) + if support_block: + text.extend(support_block) + if doc.pop('action', False): text.append(" * note: %s\n" % "This module has a corresponding action plugin.") @@ -435,4 +469,9 @@ class DocCLI(CLI): text.append(self._dump_yaml({k.upper(): doc[k]}, opt_indent)) text.append('') + metadata_block = self.get_metadata_block(doc) + if metadata_block: + text.extend(metadata_block) + text.append('') + return "\n".join(text) diff --git a/lib/ansible/parsing/metadata.py b/lib/ansible/parsing/metadata.py index f22570612d2..c021cc2c2e1 100644 --- a/lib/ansible/parsing/metadata.py +++ b/lib/ansible/parsing/metadata.py @@ -27,6 +27,11 @@ import yaml from ansible.module_utils._text import to_text +# There are currently defaults for all metadata fields so we can add it +# automatically if a file doesn't specify it +DEFAULT_METADATA = {'metadata_version': '1.1', 'status': ['preview'], 'supported_by': 'community'} + + class ParseError(Exception): """Thrown when parsing a file fails""" pass diff --git a/test/integration/targets/module_precedence/lib_no_extension/ping b/test/integration/targets/module_precedence/lib_no_extension/ping index b5bee2044cc..e30706e8946 100644 --- a/test/integration/targets/module_precedence/lib_no_extension/ping +++ b/test/integration/targets/module_precedence/lib_no_extension/ping @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/integration/targets/module_precedence/lib_with_extension/ping.py b/test/integration/targets/module_precedence/lib_with_extension/ping.py index b5bee2044cc..e30706e8946 100644 --- a/test/integration/targets/module_precedence/lib_with_extension/ping.py +++ b/test/integration/targets/module_precedence/lib_with_extension/ping.py @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/integration/targets/module_precedence/multiple_roles/bar/library/ping.py b/test/integration/targets/module_precedence/multiple_roles/bar/library/ping.py index 17f1f810b1b..e7776001e24 100644 --- a/test/integration/targets/module_precedence/multiple_roles/bar/library/ping.py +++ b/test/integration/targets/module_precedence/multiple_roles/bar/library/ping.py @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/integration/targets/module_precedence/multiple_roles/foo/library/ping.py b/test/integration/targets/module_precedence/multiple_roles/foo/library/ping.py index d12c6ce26ec..a6d153baf2a 100644 --- a/test/integration/targets/module_precedence/multiple_roles/foo/library/ping.py +++ b/test/integration/targets/module_precedence/multiple_roles/foo/library/ping.py @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/integration/targets/module_precedence/roles_no_extension/foo/library/ping b/test/integration/targets/module_precedence/roles_no_extension/foo/library/ping index d12c6ce26ec..a6d153baf2a 100644 --- a/test/integration/targets/module_precedence/roles_no_extension/foo/library/ping +++ b/test/integration/targets/module_precedence/roles_no_extension/foo/library/ping @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/integration/targets/module_precedence/roles_with_extension/foo/library/ping.py b/test/integration/targets/module_precedence/roles_with_extension/foo/library/ping.py index d12c6ce26ec..a6d153baf2a 100644 --- a/test/integration/targets/module_precedence/roles_with_extension/foo/library/ping.py +++ b/test/integration/targets/module_precedence/roles_with_extension/foo/library/ping.py @@ -20,7 +20,7 @@ # along with Ansible. If not, see . -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} diff --git a/test/sanity/validate-modules/main.py b/test/sanity/validate-modules/main.py index c348f974a4d..b0d1e1a7588 100755 --- a/test/sanity/validate-modules/main.py +++ b/test/sanity/validate-modules/main.py @@ -41,7 +41,7 @@ from ansible.utils.plugin_docs import BLACKLIST, get_docstring from module_args import get_argument_spec -from schema import doc_schema, metadata_schema, return_schema +from schema import doc_schema, metadata_1_1_schema, return_schema from utils import CaptureStd, parse_yaml from voluptuous.humanize import humanize_error @@ -892,7 +892,7 @@ class ModuleValidator(Validator): ) if metadata: - self._validate_docs_schema(metadata, metadata_schema(deprecated), + self._validate_docs_schema(metadata, metadata_1_1_schema(deprecated), 'ANSIBLE_METADATA', 316) return doc_info diff --git a/test/sanity/validate-modules/schema.py b/test/sanity/validate-modules/schema.py index f023d574518..ae94493ee9d 100644 --- a/test/sanity/validate-modules/schema.py +++ b/test/sanity/validate-modules/schema.py @@ -104,7 +104,7 @@ def doc_schema(module_name): ) -def metadata_schema(deprecated): +def metadata_1_0_schema(deprecated): valid_status = Any('stableinterface', 'preview', 'deprecated', 'removed') if deprecated: valid_status = Any('deprecated') @@ -118,6 +118,20 @@ def metadata_schema(deprecated): ) +def metadata_1_1_schema(deprecated): + valid_status = Any('stableinterface', 'preview', 'deprecated', 'removed') + if deprecated: + valid_status = Any('deprecated') + + return Schema( + { + Required('status'): [valid_status], + Required('metadata_version'): '1.1', + Required('supported_by'): Any('core', 'community', 'certified', 'network') + } + ) + + # Things to add soon #################### # 1) Recursively validate `type: complex` fields diff --git a/test/units/parsing/test_metadata.py b/test/units/parsing/test_metadata.py index 72adc845c1e..66223385675 100644 --- a/test/units/parsing/test_metadata.py +++ b/test/units/parsing/test_metadata.py @@ -41,14 +41,14 @@ from foo import bar """ STANDARD_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} """ TEXT_STD_METADATA = b""" ANSIBLE_METADATA = u''' -metadata_version: '1.0' +metadata_version: '1.1' status: - 'stableinterface' supported_by: 'core' @@ -57,7 +57,7 @@ supported_by: 'core' BYTES_STD_METADATA = b""" ANSIBLE_METADATA = b''' -metadata_version: '1.0' +metadata_version: '1.1' status: - 'stableinterface' supported_by: 'core' @@ -65,45 +65,45 @@ supported_by: 'core' """ TRAILING_COMMENT_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} # { Testing } """ MULTIPLE_STATEMENTS_METADATA = b""" -DOCUMENTATION = "" ; ANSIBLE_METADATA = {'metadata_version': '1.0', +DOCUMENTATION = "" ; ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} ; RETURNS = "" """ EMBEDDED_COMMENT_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0', +ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], # { Testing } 'supported_by': 'core'} """ HASH_SYMBOL_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0 # 4', +ANSIBLE_METADATA = {'metadata_version': '1.1 # 4', 'status': ['stableinterface'], 'supported_by': 'core # Testing '} """ HASH_SYMBOL_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0 # 4', +ANSIBLE_METADATA = {'metadata_version': '1.1 # 4', 'status': ['stableinterface'], 'supported_by': 'core # Testing '} """ HASH_COMBO_METADATA = b""" -ANSIBLE_METADATA = {'metadata_version': '1.0 # 4', +ANSIBLE_METADATA = {'metadata_version': '1.1 # 4', 'status': ['stableinterface'], # { Testing } 'supported_by': 'core'} # { Testing } """ -METADATA = {'metadata_version': '1.0', 'status': ['stableinterface'], 'supported_by': 'core'} -HASH_SYMBOL_METADATA = {'metadata_version': '1.0 # 4', 'status': ['stableinterface'], 'supported_by': 'core'} +METADATA = {'metadata_version': '1.1', 'status': ['stableinterface'], 'supported_by': 'core'} +HASH_SYMBOL_METADATA = {'metadata_version': '1.1 # 4', 'status': ['stableinterface'], 'supported_by': 'core'} METADATA_EXAMPLES = ( # Standard import @@ -225,15 +225,15 @@ def test_module_data_param_given_with_offset(): def test_invalid_dict_metadata(): with pytest.raises(SyntaxError): - assert md.extract_metadata(module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.0",\n' + REGULAR_IMPORTS) + assert md.extract_metadata(module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.1",\n' + REGULAR_IMPORTS) with pytest.raises(md.ParseError, message='Unable to find the end of dictionary'): - assert md.extract_metadata(module_ast=ast.parse(LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.0"}\n' + REGULAR_IMPORTS), - module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.0",\n' + REGULAR_IMPORTS, + assert md.extract_metadata(module_ast=ast.parse(LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.1"}\n' + REGULAR_IMPORTS), + module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.1",\n' + REGULAR_IMPORTS, offsets=True) def test_multiple_statements_limitation(): with pytest.raises(md.ParseError, message='Multiple statements per line confuses the module metadata parser.'): - assert md.extract_metadata(module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.0"}; a=b\n' + REGULAR_IMPORTS, + assert md.extract_metadata(module_data=LICENSE + FUTURE_IMPORTS + b'ANSIBLE_METADATA={"metadata_version": "1.1"}; a=b\n' + REGULAR_IMPORTS, offsets=True)