Move common build code from _build_helpers (#55986)

We have some common code used by several docs scripts.  Migrate that
into the build-only shared code repository.

* Move lib/ansible/utils/_build_helpers.py to the directory for common
  build code
* Migrate docs/bin/dump_config.py to a build-ansible subcommand
* Migrate dump_keywords to the build-ansible framework
  * Make the script more maintainable by using functions and good
    variable names
  * Port to Python3 idioms
  * Fix bug so that private attributes will be undocumented
* Move generate_man to a build-ansible subcommand
* Port plugin_formatter to a build-ansible subcommand
* Rework command_plugins so that docs scripts can target Python-3.4+ and
  releng-only subcommands can use more recent versions of Python.
  The architecture is now that command_plugins/* need to be importable
  on Python-3.4.  The init_parsers() method needs to run on Python-3.4.
  But the main() method can utilize features of more recent Python as
  long as it fits within those parameters.
* Update docs build requirements

Port the plugin_formatter to build-ansible framework

Makefile

@@ -33,7 +33,7 @@ ASCII2MAN = @echo "ERROR: rst2man from docutils command is not installed but is
 endif
 
 PYTHON=python
-GENERATE_CLI = $(PYTHON) docs/bin/generate_man.py
+GENERATE_CLI = hacking/build-ansible.py generate-man
 
 SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
 

docs/bin/dump_config.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python
-
-import optparse
-import os
-import sys
-import yaml
-
-from jinja2 import Environment, FileSystemLoader
-from ansible.module_utils._text import to_bytes
-from ansible.utils._build_helpers import update_file_if_different
-
-DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
-
-
-def generate_parser():
-    p = optparse.OptionParser(
-        version='%prog 1.0',
-        usage='usage: %prog [options]',
-        description='Generate module documentation from metadata',
-    )
-    p.add_option("-t", "--template-file", action="store", dest="template_file", default=DEFAULT_TEMPLATE_FILE, help="directory containing Jinja2 templates")
-    p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
-    p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
-
-    (options, args) = p.parse_args()
-
-    return p
-
-
-def fix_description(config_options):
-    '''some descriptions are strings, some are lists. workaround it...'''
-
-    for config_key in config_options:
-        description = config_options[config_key].get('description', [])
-        if isinstance(description, list):
-            desc_list = description
-        else:
-            desc_list = [description]
-        config_options[config_key]['description'] = desc_list
-    return config_options
-
-
-def main(args):
-
-    parser = generate_parser()
-    (options, args) = parser.parse_args()
-
-    output_dir = os.path.abspath(options.output_dir)
-    template_file_full_path = os.path.abspath(options.template_file)
-    template_file = os.path.basename(template_file_full_path)
-    template_dir = os.path.dirname(os.path.abspath(template_file_full_path))
-
-    if options.docs:
-        with open(options.docs) as f:
-            docs = yaml.safe_load(f)
-    else:
-        docs = {}
-
-    config_options = docs
-    config_options = fix_description(config_options)
-
-    env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
-    template = env.get_template(template_file)
-    output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
-    temp_vars = {'config_options': config_options}
-
-    data = to_bytes(template.render(temp_vars))
-    update_file_if_different(output_name, data)
-
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[:]))

docs/bin/dump_keywords.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python
-
-import optparse
-import re
-from distutils.version import LooseVersion
-
-import jinja2
-import yaml
-from jinja2 import Environment, FileSystemLoader
-
-from ansible.module_utils._text import to_bytes
-from ansible.playbook import Play
-from ansible.playbook.block import Block
-from ansible.playbook.role import Role
-from ansible.playbook.task import Task
-from ansible.utils._build_helpers import update_file_if_different
-
-template_file = 'playbooks_keywords.rst.j2'
-oblist = {}
-clist = []
-class_list = [Play, Role, Block, Task]
-
-p = optparse.OptionParser(
-    version='%prog 1.0',
-    usage='usage: %prog [options]',
-    description='Generate playbook keyword documentation from code and descriptions',
-)
-p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="../templates", help="directory containing Jinja2 templates")
-p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
-p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
-
-(options, args) = p.parse_args()
-
-for aclass in class_list:
-    aobj = aclass()
-    name = type(aobj).__name__
-
-    if options.docs:
-        with open(options.docs) as f:
-            docs = yaml.safe_load(f)
-    else:
-        docs = {}
-
-    # build ordered list to loop over and dict with attributes
-    clist.append(name)
-    oblist[name] = dict((x, aobj.__dict__['_attributes'][x]) for x in aobj.__dict__['_attributes'] if 'private' not in x or not x.private)
-
-    # pick up docs if they exist
-    for a in oblist[name]:
-        if a in docs:
-            oblist[name][a] = docs[a]
-        else:
-            # check if there is an alias, otherwise undocumented
-            alias = getattr(getattr(aobj, '_%s' % a), 'alias', None)
-            if alias and alias in docs:
-                oblist[name][alias] = docs[alias]
-                del oblist[name][a]
-            else:
-                oblist[name][a] = ' UNDOCUMENTED!! '
-
-    # loop is really with_ for users
-    if name == 'Task':
-        oblist[name]['with_<lookup_plugin>'] = 'The same as ``loop`` but magically adds the output of any lookup plugin to generate the item list.'
-
-    # local_action is implicit with action
-    if 'action' in oblist[name]:
-        oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``'
-
-    # remove unusable (used to be private?)
-    for nouse in ('loop_args', 'loop_with'):
-        if nouse in oblist[name]:
-            del oblist[name][nouse]
-
-env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
-template = env.get_template(template_file)
-outputname = options.output_dir + template_file.replace('.j2', '')
-tempvars = {'oblist': oblist, 'clist': clist}
-
-keyword_page = template.render(tempvars)
-if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
-    # jinja2 < 2.10's indent filter indents blank lines.  Cleanup
-    keyword_page = re.sub(' +\n', '\n', keyword_page)
-
-update_file_if_different(outputname, to_bytes(keyword_page))

docs/docsite/Makefile

@@ -1,10 +1,10 @@
 OS := $(shell uname -s)
 SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
-FORMATTER=../bin/plugin_formatter.py
+PLUGIN_FORMATTER=../../hacking/build-ansible.py document-plugins
 TESTING_FORMATTER=../bin/testing_formatter.sh
-DUMPER=../bin/dump_keywords.py
+KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords
-CONFIG_DUMPER=../bin/dump_config.py
+CONFIG_DUMPER=../../hacking/build-ansible.py document-config
-GENERATE_CLI=../bin/generate_man.py
+GENERATE_CLI=../../hacking/build-ansible.py generate-man
 ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
 CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
 else
@@ -79,24 +79,24 @@ clean:
 .PHONY: docs clean
 
 # TODO: make generate_man output dir cli option
-cli: $(GENERATE_CLI)
+cli:
 	mkdir -p rst/cli
 	PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py
 
-keywords: $(FORMATTER) ../templates/playbooks_keywords.rst.j2
+keywords: ../templates/playbooks_keywords.rst.j2
-	PYTHONPATH=../../lib $(DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
+	PYTHONPATH=../../lib $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
 
-config:
+config: ../templates/config.rst.j2
 	PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml
 
-modules: $(FORMATTER) ../templates/plugin.rst.j2
+modules: ../templates/plugin.rst.j2
-	PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
+	PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
 
-plugins: $(FORMATTER) ../templates/plugin.rst.j2
+plugins: ../templates/plugin.rst.j2
 	@echo "looping over doc plugins"
 	for plugin in $(DOC_PLUGINS); \
 	do \
-		PYTHONPATH=../../lib $(FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
+		PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
 	done
 
 testing:

docs/docsite/requirements.txt

@@ -5,3 +5,4 @@ rstcheck
 sphinx
 sphinx-notfound-page
 Pygments >= 2.4.0
+straight.plugin # Needed for hacking/build-ansible.py which is the backend build script

docs/docsite/rst/community/documentation_contributions.rst

@@ -60,7 +60,8 @@ If you make multiple changes to the documentation, or add more than a line to it
 #. Test your changes for rST errors.
 #. Build the page, and preferably the entire documentation site, locally.
 
-To work with documentation on your local machine, you need the following packages installed:
+To work with documentation on your local machine, you need to have python-3.5 or greater and the
+following packages installed:
 
 - gcc
 - jinja2
@@ -72,6 +73,7 @@ To work with documentation on your local machine, you need the following package
 - six
 - sphinx
 - sphinx-notfound-page
+- straight.plugin
 
 .. note::
 

docs/templates/playbooks_keywords.rst.j2

@@ -19,15 +19,15 @@ These are the keywords available on common playbook objects. Keywords are one of
    :local:
    :depth: 1
 
-{% for name in clist %}
+{% for name in playbook_class_names %}
 
 {{ name }}
 {{ '-' * name|length }}
 .. glossary::
 
-{% for attribute in oblist[name]|sort %}
+{% for attribute in pb_keywords[name]|sort %}
     {{ attribute }}
-        {{ oblist[name][attribute] |indent(8) }}
+        {{ pb_keywords[name][attribute] |indent(8) }}
 
 {% endfor %}
 {% endfor %}

hacking/build-ansible.py

@@ -21,18 +21,17 @@ except ImportError:
     argcomplete = None
 
 
-def set_sys_path(this_script=__file__):
+def build_lib_path(this_script=__file__):
-    """Add path to the common librarydirectory to :attr:`sys.path`"""
+    """Return path to the common build library directory"""
     hacking_dir = os.path.dirname(this_script)
     libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
 
-    if libdir not in sys.path:
+    return libdir
-        sys.path.insert(0, libdir)
 
 
-set_sys_path()
+sys.path.insert(0, build_lib_path())
 
-from build_ansible import commands
+from build_ansible import commands, errors
 
 
 def create_arg_parser(program_name):
@@ -63,13 +62,26 @@ def main():
         argcomplete.autocomplete(arg_parser)
 
     args = arg_parser.parse_args(sys.argv[1:])
+    if args.command is None:
+        print('Please specify a subcommand to run')
+        sys.exit(1)
 
     for subcommand in subcommands:
         if subcommand.name == args.command:
-            sys.exit(subcommand.main(args))
+            command = subcommand
+            break
+    else:
+        # Note: We should never trigger this because argparse should shield us from it
+        print('Error: {0} was not a recognized subcommand'.format(args.command))
+        sys.exit(1)
 
-    print('Error: Select a subcommand')
+    try:
-    arg_parser.print_usage()
+        retval = command.main(args)
+    except errors.DependencyError as e:
+        print(e)
+        sys.exit(2)
+
+    sys.exit(retval)
 
 
 if __name__ == '__main__':

hacking/build_library/build_ansible/command_plugins/dump_config.py

@@ -0,0 +1,80 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import os.path
+import pathlib
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+from ansible.module_utils._text import to_bytes
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
+
+DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
+DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
+
+
+def fix_description(config_options):
+    '''some descriptions are strings, some are lists. workaround it...'''
+
+    for config_key in config_options:
+        description = config_options[config_key].get('description', [])
+        if isinstance(description, list):
+            desc_list = description
+        else:
+            desc_list = [description]
+        config_options[config_key]['description'] = desc_list
+    return config_options
+
+
+class DocumentConfig(Command):
+    name = 'document-config'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description='Generate module documentation from metadata')
+        parser.add_argument("-t", "--template-file", action="store", dest="template_file",
+                            default=DEFAULT_TEMPLATE_FILE,
+                            help="Jinja2 template to use for the config")
+        parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
+                            default=DEFAULT_TEMPLATE_DIR,
+                            help="directory containing Jinja2 templates")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
+                            help="Output directory for rst files")
+        parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
+                            help="Source for attribute docs")
+
+    @staticmethod
+    def main(args):
+        output_dir = os.path.abspath(args.output_dir)
+        template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
+        template_file = os.path.basename(template_file_full_path)
+        template_dir = os.path.dirname(template_file_full_path)
+
+        if args.docs:
+            with open(args.docs) as f:
+                docs = yaml.safe_load(f)
+        else:
+            docs = {}
+
+        config_options = docs
+        config_options = fix_description(config_options)
+
+        env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
+        template = env.get_template(template_file)
+        output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
+        temp_vars = {'config_options': config_options}
+
+        data = to_bytes(template.render(temp_vars))
+        update_file_if_different(output_name, data)
+
+        return 0

hacking/build_library/build_ansible/command_plugins/dump_keywords.py

@@ -0,0 +1,125 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import importlib
+import os.path
+import pathlib
+import re
+from distutils.version import LooseVersion
+
+import jinja2
+import yaml
+from jinja2 import Environment, FileSystemLoader
+
+from ansible.module_utils._text import to_bytes
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
+
+
+DEFAULT_TEMPLATE_DIR = str(pathlib.Path(__file__).resolve().parents[4] / 'docs/templates')
+TEMPLATE_FILE = 'playbooks_keywords.rst.j2'
+PLAYBOOK_CLASS_NAMES = ['Play', 'Role', 'Block', 'Task']
+
+
+def load_definitions(keyword_definitions_file):
+    docs = {}
+    with open(keyword_definitions_file) as f:
+        docs = yaml.safe_load(f)
+
+    return docs
+
+
+def extract_keywords(keyword_definitions):
+    pb_keywords = {}
+    for pb_class_name in PLAYBOOK_CLASS_NAMES:
+        if pb_class_name == 'Play':
+            module_name = 'ansible.playbook'
+        else:
+            module_name = 'ansible.playbook.{0}'.format(pb_class_name.lower())
+        module = importlib.import_module(module_name)
+        playbook_class = getattr(module, pb_class_name, None)
+        if playbook_class is None:
+            raise ImportError("We weren't able to import the module {0}".format(module_name))
+
+        # Maintain order of the actual class names for our output
+        # Build up a mapping of playbook classes to the attributes that they hold
+        pb_keywords[pb_class_name] = {k: v for (k, v) in playbook_class._valid_attrs.items()
+                                      # Filter private attributes as they're not usable in playbooks
+                                      if not v.private}
+
+        # pick up definitions if they exist
+        for keyword in tuple(pb_keywords[pb_class_name]):
+            if keyword in keyword_definitions:
+                pb_keywords[pb_class_name][keyword] = keyword_definitions[keyword]
+            else:
+                # check if there is an alias, otherwise undocumented
+                alias = getattr(getattr(playbook_class, '_%s' % keyword), 'alias', None)
+                if alias and alias in keyword_definitions:
+                    pb_keywords[pb_class_name][alias] = keyword_definitions[alias]
+                    del pb_keywords[pb_class_name][keyword]
+                else:
+                    pb_keywords[pb_class_name][keyword] = ' UNDOCUMENTED!! '
+
+        # loop is really with_ for users
+        if pb_class_name == 'Task':
+            pb_keywords[pb_class_name]['with_<lookup_plugin>'] = (
+                'The same as ``loop`` but magically adds the output of any lookup plugin to'
+                ' generate the item list.')
+
+        # local_action is implicit with action
+        if 'action' in pb_keywords[pb_class_name]:
+            pb_keywords[pb_class_name]['local_action'] = ('Same as action but also implies'
+                                                          ' ``delegate_to: localhost``')
+
+    return pb_keywords
+
+
+def generate_page(pb_keywords, template_dir):
+    env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
+    template = env.get_template(TEMPLATE_FILE)
+    tempvars = {'pb_keywords': pb_keywords, 'playbook_class_names': PLAYBOOK_CLASS_NAMES}
+
+    keyword_page = template.render(tempvars)
+    if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
+        # jinja2 < 2.10's indent filter indents blank lines.  Cleanup
+        keyword_page = re.sub(' +\n', '\n', keyword_page)
+
+    return keyword_page
+
+
+class DocumentKeywords(Command):
+    name = 'document-keywords'
+
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(cls.name, description='Generate playbook keyword documentation from'
+                            ' code and descriptions')
+        parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
+                            default=DEFAULT_TEMPLATE_DIR,
+                            help="directory containing Jinja2 templates")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
+                            default='/tmp/', help="Output directory for rst files")
+        parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
+                            help="Source for attribute docs")
+
+    @staticmethod
+    def main(args):
+        if not args.docs:
+            print('Definitions for keywords must be specified via `--docs-source FILENAME`')
+            return 1
+
+        keyword_definitions = load_definitions(args.docs)
+        pb_keywords = extract_keywords(keyword_definitions)
+
+        keyword_page = generate_page(pb_keywords, args.template_dir)
+        outputname = os.path.join(args.output_dir, TEMPLATE_FILE.replace('.j2', ''))
+        update_file_if_different(outputname, to_bytes(keyword_page))
+
+        return 0

hacking/build_library/build_ansible/command_plugins/generate_man.py

@@ -1,25 +1,27 @@
-#!/usr/bin/env python
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
 
 import argparse
-import os
+import os.path
+import pathlib
 import sys
 
 from jinja2 import Environment, FileSystemLoader
 
 from ansible.module_utils._text import to_bytes
-from ansible.utils._build_helpers import update_file_if_different
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
 
 
-def generate_parser():
+DEFAULT_TEMPLATE_FILE = pathlib.Path(__file__).parents[4] / 'docs/templates/man.j2'
-    p = argparse.ArgumentParser(
-        description='Generate cli documentation from cli docstrings',
-    )
-
-    p.add_argument("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
-    p.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
-    p.add_argument("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
-    p.add_argument('args', help='CLI module(s)', metavar='module', nargs='*')
-    return p
 
 
 # from https://www.python.org/dev/peps/pep-0257/
@@ -213,27 +215,38 @@ def opts_docs(cli_class_name, cli_module_name):
     return docs
 
 
-if __name__ == '__main__':
+class GenerateMan(Command):
+    name = 'generate-man'
 
-    parser = generate_parser()
+    @classmethod
+    def init_parser(cls, add_parser):
+        parser = add_parser(name=cls.name,
+                            description='Generate cli documentation from cli docstrings')
 
-    options = parser.parse_args()
+        parser.add_argument("-t", "--template-file", action="store", dest="template_file",
+                            default=DEFAULT_TEMPLATE_FILE, help="path to jinja2 template")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
+                            default='/tmp/', help="Output directory for rst files")
+        parser.add_argument("-f", "--output-format", action="store", dest="output_format",
+                            default='man',
+                            help="Output format for docs (the default 'man' or 'rst')")
+        parser.add_argument('cli_modules', help='CLI module name(s)', metavar='MODULE_NAME', nargs='*')
 
-    template_file = options.template_file
+    @staticmethod
+    def main(args):
+        template_file = args.template_file
         template_path = os.path.expanduser(template_file)
         template_dir = os.path.abspath(os.path.dirname(template_path))
         template_basename = os.path.basename(template_file)
 
-    output_dir = os.path.abspath(options.output_dir)
+        output_dir = os.path.abspath(args.output_dir)
-    output_format = options.output_format
+        output_format = args.output_format
 
-    cli_modules = options.args
+        cli_modules = args.cli_modules
 
         # various cli parsing things checks sys.argv if the 'args' that are passed in are []
         # so just remove any args so the cli modules dont try to parse them resulting in warnings
         sys.argv = [sys.argv[0]]
-    # need to be in right dir
-    os.chdir(os.path.dirname(__file__))
 
         allvars = {}
         output = {}

hacking/build_library/build_ansible/command_plugins/plugin_formatter.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
 # Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
 # Copyright: (c) 2017, Ansible Project
@@ -45,7 +44,10 @@ from ansible.module_utils.six import iteritems, string_types
 from ansible.plugins.loader import fragment_loader
 from ansible.utils import plugin_docs
 from ansible.utils.display import Display
-from ansible.utils._build_helpers import update_file_if_different
+
+# Pylint doesn't understand Python3 namespace modules.
+from ..change_detection import update_file_if_different  # pylint: disable=relative-beyond-top-level
+from ..commands import Command  # pylint: disable=relative-beyond-top-level
 
 
 #####################################################################################
@@ -363,29 +365,6 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
     return module_info, categories
 
 
-def generate_parser():
-    ''' generate an optparse parser '''
-
-    p = optparse.OptionParser(
-        version='%prog 1.0',
-        usage='usage: %prog [options] arg1 arg2',
-        description='Generate module documentation from metadata',
-    )
-
-    p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
-    p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
-    p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)")
-    p.add_option("-T", "--template-dir", action="append", dest="template_dir", help="directory containing Jinja2 templates")
-    p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
-    p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
-    p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
-    p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None,
-                 help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.")
-    p.add_option('-V', action='version', help='Show version number and exit')
-    p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)")
-    return p
-
-
 def jinja2_environment(template_dir, typ, plugin_type):
 
     env = Environment(loader=FileSystemLoader(template_dir),
@@ -734,40 +713,66 @@ def validate_options(options):
         sys.exit("--template-dir must be specified")
 
 
-def main():
+class DocumentPlugins(Command):
+    name = 'document-plugins'
 
-    # INIT
+    @classmethod
-    p = generate_parser()
+    def init_parser(cls, add_parser):
-    (options, args) = p.parse_args()
+        parser = add_parser(cls.name, description='Generate module documentation from metadata')
-    if not options.template_dir:
+
-        options.template_dir = ["hacking/templates"]
+        parser.add_argument("-A", "--ansible-version", action="store", dest="ansible_version",
-    validate_options(options)
+                            default="unknown", help="Ansible version number")
-    display.verbosity = options.verbosity
+        parser.add_argument("-M", "--module-dir", action="store", dest="module_dir",
-    plugin_type = options.plugin_type
+                            default=MODULEDIR, help="Ansible library path")
+        parser.add_argument("-P", "--plugin-type", action="store", dest="plugin_type",
+                            default='module', help="The type of plugin (module, lookup, etc)")
+        parser.add_argument("-T", "--template-dir", action="append", dest="template_dir",
+                            help="directory containing Jinja2 templates")
+        parser.add_argument("-t", "--type", action='store', dest='type', choices=['rst'],
+                            default='rst', help="Document type")
+        parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default=None,
+                            help="Output directory for module files")
+        parser.add_argument("-I", "--includes-file", action="store", dest="includes_file",
+                            default=None, help="Create a file containing list of processed modules")
+        parser.add_argument("-l", "--limit-to-modules", '--limit-to', action="store",
+                            dest="limit_to", default=None, help="Limit building module documentation"
+                            " to comma-separated list of plugins. Specify non-existing plugin name"
+                            " for no plugins.")
+        parser.add_argument('-V', action='version', help='Show version number and exit')
+        parser.add_argument('-v', '--verbose', dest='verbosity', default=0, action="count",
+                            help="verbose mode (increase number of 'v's for more)")
+
+    @staticmethod
+    def main(args):
+        if not args.template_dir:
+            args.template_dir = ["hacking/templates"]
+        validate_options(args)
+        display.verbosity = args.verbosity
+        plugin_type = args.plugin_type
 
         display.display("Evaluating %s files..." % plugin_type)
 
         # prep templating
-    templates = jinja2_environment(options.template_dir, options.type, plugin_type)
+        templates = jinja2_environment(args.template_dir, args.type, plugin_type)
 
         # set file/directory structure
         if plugin_type == 'module':
             # trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
             outputname = '%s_' + '%s.rst' % plugin_type
-        output_dir = options.output_dir
+            output_dir = args.output_dir
         else:
             # for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
             outputname = '%s.rst'
-        output_dir = '%s/plugins/%s' % (options.output_dir, plugin_type)
+            output_dir = '%s/plugins/%s' % (args.output_dir, plugin_type)
 
         display.vv('output name: %s' % outputname)
         display.vv('output dir: %s' % output_dir)
 
         # Convert passed-in limit_to to None or list of modules.
-    if options.limit_to is not None:
+        if args.limit_to is not None:
-        options.limit_to = [s.lower() for s in options.limit_to.split(",")]
+            args.limit_to = [s.lower() for s in args.limit_to.split(",")]
 
-    plugin_info, categories = get_plugin_info(options.module_dir, limit_to=options.limit_to, verbose=(options.verbosity > 0))
+        plugin_info, categories = get_plugin_info(args.module_dir, limit_to=args.limit_to, verbose=(args.verbosity > 0))
 
         categories['all'] = {'_modules': plugin_info.keys()}
 
@@ -777,7 +782,7 @@ def main():
             display.vvvvv(pp.pformat(plugin_info))
 
         # Transform the data
-    if options.type == 'rst':
+        if args.type == 'rst':
             display.v('Generating rst')
             for key, record in plugin_info.items():
                 display.vv(key)
@@ -799,7 +804,7 @@ def main():
 
         # Render all the individual plugin pages
         display.v('Generating plugin pages')
-    process_plugins(plugin_info, templates, outputname, output_dir, options.ansible_version, plugin_type)
+        process_plugins(plugin_info, templates, outputname, output_dir, args.ansible_version, plugin_type)
 
         # Render all the categories for modules
         if plugin_type == 'module':
@@ -810,6 +815,4 @@ def main():
             # Render all the categories for modules
             process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
 
-
+        return 0
-if __name__ == '__main__':
-    main()

hacking/build_library/build_ansible/command_plugins/porting_guide.py

@@ -121,7 +121,7 @@ def generate_porting_guide(version):
 
 
 def write_guide(version, guide_content):
-    filename = f'porting_guide_{version}.rst'
+    filename = 'porting_guide_{0}.rst'.format(version)
     with open(filename, 'w') as out_file:
         out_file.write(guide_content)
 

hacking/build_library/build_ansible/command_plugins/release_announcement.py

@@ -7,148 +7,13 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
-import argparse
-import asyncio
-import datetime
-import hashlib
-import os.path
 import sys
 from collections import UserString
 from distutils.version import LooseVersion
 
-import aiohttp
-from jinja2 import Environment, DictLoader
-
 # Pylint doesn't understand Python3 namespace modules.
 from ..commands import Command  # pylint: disable=relative-beyond-top-level
-
+from .. import errors  # pylint: disable=relative-beyond-top-level
-
-# pylint: disable=
-VERSION_FRAGMENT = """
-{%- if versions | length > 1 %}
-  {% for version in versions %}
-    {% if loop.last %}and {{ version }}{% else %}
-      {% if versions | length == 2 %}{{ version }} {% else %}{{ version }}, {% endif -%}
-    {% endif -%}
-  {% endfor -%}
-{%- else %}{{ versions[0] }}{% endif -%}
-"""
-
-LONG_TEMPLATE = """
-{% set plural = False if versions | length == 1 else True %}
-{% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
-
-To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
-Subject: New Ansible release{% if plural %}s{% endif %} {{ version_str }}
-
-{% filter wordwrap %}
-Hi all- we're happy to announce that the general release of Ansible {{ version_str }}{% if plural %} are{%- else %} is{%- endif %} now available!
-{% endfilter %}
-
-
-
-How do you get it?
-------------------
-
-{% for version in versions %}
-$ pip install ansible=={{ version }} --user
-{% if not loop.last %}
-or
-{% endif %}
-{% endfor %}
-
-The tar.gz of the release{% if plural %}s{% endif %} can be found here:
-
-{% for version in versions %}
-* {{ version }}
-  https://releases.ansible.com/ansible/ansible-{{ version }}.tar.gz
-  SHA256: {{ hashes[version] }}
-{% endfor %}
-
-
-What's new in {{ version_str }}
-{{ '-' * (14 + version_str | length) }}
-
-{% filter wordwrap %}
-{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
-{% endfilter %}
-
-
-{% for version in versions %}
-* {{ version }}
-  https://github.com/ansible/ansible/blob/stable-{{ version.split('.')[:2] | join('.') }}/changelogs/CHANGELOG-v{{ version.split('.')[:2] | join('.') }}.rst
-{% endfor %}
-
-
-What's the schedule for future maintenance releases?
-----------------------------------------------------
-
-{% filter wordwrap %}
-Future maintenance releases will occur approximately every 3 weeks.  So expect the next one around {{ next_release.strftime('%Y-%m-%d') }}.
-{% endfilter %}
-
-
-
-Porting Help
-------------
-
-{% filter wordwrap %}
-We've published a porting guide at
-https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_{{ latest_ver.split('.')[:2] | join('.') }}.html to help migrate your content to {{ latest_ver.split('.')[:2] | join('.') }}.
-{% endfilter %}
-
-
-
-{% filter wordwrap %}
-If you discover any errors or if any of your working playbooks break when you upgrade to {{ latest_ver }}, please use the following link to report the regression:
-{% endfilter %}
-
-
-  https://github.com/ansible/ansible/issues/new/choose
-
-{% filter wordwrap %}
-In your issue, be sure to mention the Ansible version that works and the one that doesn't.
-{% endfilter %}
-
-
-Thanks!
-
--{{ name }}
-
-"""  # noqa for E501 (line length).
-# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
-# proper wrapping to occur
-
-SHORT_TEMPLATE = """
-{% set plural = False if versions | length == 1 else True %}
-@ansible
-{{ version_str }}
-{% if plural %}
-  have
-{% else %}
-  has
-{% endif %}
-been released! Get
-{% if plural %}
-them
-{% else %}
-it
-{% endif %}
-on PyPI: pip install ansible=={{ (versions|sort(attribute='ver_obj'))[-1] }},
-https://releases.ansible.com/ansible/, the Ansible PPA on Launchpad, or GitHub.  Happy automating!
-"""  # noqa for E501 (line length).
-# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
-# proper wrapping to occur
-
-JINJA_ENV = Environment(
-    loader=DictLoader({'long': LONG_TEMPLATE,
-                       'short': SHORT_TEMPLATE,
-                       'version_string': VERSION_FRAGMENT,
-                       }),
-    extensions=['jinja2.ext.i18n'],
-    trim_blocks=True,
-    lstrip_blocks=True,
-)
 
 
 class VersionStr(UserString):
@@ -167,108 +32,6 @@ def transform_args(args):
     return args
 
 
-async def calculate_hash_from_tarball(session, version):
-    tar_url = f'https://releases.ansible.com/ansible/ansible-{version}.tar.gz'
-    tar_task = asyncio.create_task(session.get(tar_url))
-    tar_response = await tar_task
-
-    tar_hash = hashlib.sha256()
-    while True:
-        chunk = await tar_response.content.read(1024)
-        if not chunk:
-            break
-        tar_hash.update(chunk)
-
-    return tar_hash.hexdigest()
-
-
-async def parse_hash_from_file(session, version):
-    filename = f'ansible-{version}.tar.gz'
-    hash_url = f'https://releases.ansible.com/ansible/{filename}.sha'
-    hash_task = asyncio.create_task(session.get(hash_url))
-    hash_response = await hash_task
-
-    hash_content = await hash_response.read()
-    precreated_hash, precreated_filename = hash_content.split(None, 1)
-    if filename != precreated_filename.strip().decode('utf-8'):
-        raise ValueError(f'Hash file contains hash for a different file: {precreated_filename}')
-
-    return precreated_hash.decode('utf-8')
-
-
-async def get_hash(session, version):
-    calculated_hash = await calculate_hash_from_tarball(session, version)
-    precreated_hash = await parse_hash_from_file(session, version)
-
-    if calculated_hash != precreated_hash:
-        raise ValueError(f'Hash in file ansible-{version}.tar.gz.sha {precreated_hash} does not'
-                         f' match hash of tarball {calculated_hash}')
-
-    return calculated_hash
-
-
-async def get_hashes(versions):
-    hashes = {}
-    requestors = {}
-    async with aiohttp.ClientSession() as aio_session:
-        for version in versions:
-            requestors[version] = asyncio.create_task(get_hash(aio_session, version))
-
-        for version, request in requestors.items():
-            await request
-            hashes[version] = request.result()
-
-    return hashes
-
-
-def next_release_date(weeks=3):
-    days_in_the_future = weeks * 7
-    today = datetime.datetime.now()
-    numeric_today = today.weekday()
-
-    # We release on Thursdays
-    if numeric_today == 3:
-        # 3 is Thursday
-        pass
-    elif numeric_today == 4:
-        # If this is Friday, we can adjust back to Thursday for the next release
-        today -= datetime.timedelta(days=1)
-    elif numeric_today < 3:
-        # Otherwise, slide forward to Thursday
-        today += datetime.timedelta(days=(3 - numeric_today))
-    else:
-        # slightly different formula if it's past Thursday this week.  We need to go forward to
-        # Thursday of next week
-        today += datetime.timedelta(days=(10 - numeric_today))
-
-    next_release = today + datetime.timedelta(days=days_in_the_future)
-    return next_release
-
-
-def generate_long_message(versions, name):
-    hashes = asyncio.run(get_hashes(versions))
-
-    version_template = JINJA_ENV.get_template('version_string')
-    version_str = version_template.render(versions=versions).strip()
-
-    next_release = next_release_date()
-
-    template = JINJA_ENV.get_template('long')
-    message = template.render(versions=versions, version_str=version_str,
-                              name=name, hashes=hashes, next_release=next_release)
-    return message
-
-
-def generate_short_message(versions):
-    version_template = JINJA_ENV.get_template('version_string')
-    version_str = version_template.render(versions=versions).strip()
-
-    template = JINJA_ENV.get_template('short')
-    message = template.render(versions=versions, version_str=version_str)
-    message = ' '.join(message.split()) + '\n'
-    return message
-
-
 def write_message(filename, message):
     if filename != '-':
         with open(filename, 'w') as out_file:
@@ -294,12 +57,21 @@ class ReleaseAnnouncementCommand(Command):
         parser.add_argument("--twitter-out", type=str, default="-",
                             help="Filename to place the twitter announcement into")
 
-    @staticmethod
+    @classmethod
-    def main(args):
+    def main(cls, args):
+        if sys.version_info < (3, 6):
+            raise errors.DependencyError('The {0} subcommand needs Python-3.6+'
+                                         ' to run'.format(cls.name))
+
+        # Import here because these functions are invalid on Python-3.5 and the command plugins and
+        # init_parser() method need to be compatible with Python-3.4+ for now.
+        # Pylint doesn't understand Python3 namespace modules.
+        from .. announce import create_short_message, create_long_message  # pylint: disable=relative-beyond-top-level
+
         args = transform_args(args)
 
-        twitter_message = generate_short_message(args.versions)
+        twitter_message = create_short_message(args.versions)
-        email_message = generate_long_message(args.versions, args.name)
+        email_message = create_long_message(args.versions, args.name)
 
         write_message(args.twitter_out, twitter_message)
         write_message(args.email_out, email_message)

hacking/build_library/build_ansible/errors.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+# Copyright: (c) 2019, Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class DependencyError(Exception):
+    """Used when a dependency is unmet"""
+    pass

lib/ansible/playbook/attribute.py

@@ -52,8 +52,9 @@ class Attribute:
         :kwarg isa: The type of the attribute.  Allowable values are a string
             representation of any yaml basic datatype, python class, or percent.
             (Enforced at post-validation time).
-        :kwarg private: Hides the attribute from being documented.
+        :kwarg private: Not used at runtime.  The docs playbook keyword dumper uses it to determine
-                        TODO: it should also should prevent it from being user settable
+            that a keyword should not be documented.  mpdehaan had plans to remove attributes marked
+            private from the ds so they would not have been available at all.
         :kwarg default: Default value if unspecified in the YAML document.
         :kwarg required: Whether or not the YAML document must contain this field.
             If the attribute is None when post-validated, an error will be raised.

test/sanity/compile/python2.6-skip.txt

@@ -1,3 +1,11 @@
 # The following are only run by release engineers who can be asked to have newer Python3 on their systems
 hacking/build_library/build_ansible/command_plugins/porting_guide.py
 hacking/build_library/build_ansible/command_plugins/release_announcement.py
+
+# The following are used to build docs.  Since we explicitly say that the controller won't run on
+# Python-2.6 (docs are built controller-side) and EPEL-6, the only LTS platform with Python-2.6,
+# doesn't have a new enough sphinx to build docs, do not test these under Python-2.6
+hacking/build_library/build_ansible/command_plugins/dump_config.py
+hacking/build_library/build_ansible/command_plugins/dump_keywords.py
+hacking/build_library/build_ansible/command_plugins/generate_man.py
+hacking/build_library/build_ansible/command_plugins/plugin_formatter.py