Move common build code from _build_helpers (#55986)

We have some common code used by several docs scripts.  Migrate that
into the build-only shared code repository.

* Move lib/ansible/utils/_build_helpers.py to the directory for common
  build code
* Migrate docs/bin/dump_config.py to a build-ansible subcommand
* Migrate dump_keywords to the build-ansible framework
  * Make the script more maintainable by using functions and good
    variable names
  * Port to Python3 idioms
  * Fix bug so that private attributes will be undocumented
* Move generate_man to a build-ansible subcommand
* Port plugin_formatter to a build-ansible subcommand
* Rework command_plugins so that docs scripts can target Python-3.4+ and
  releng-only subcommands can use more recent versions of Python.
  The architecture is now that command_plugins/* need to be importable
  on Python-3.4.  The init_parsers() method needs to run on Python-3.4.
  But the main() method can utilize features of more recent Python as
  long as it fits within those parameters.
* Update docs build requirements

Port the plugin_formatter to build-ansible framework
This commit is contained in:
Toshio Kuratomi 2019-07-16 12:19:01 -07:00 committed by GitHub
parent 65e0f37fc0
commit 019d078a5a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 457 additions and 586 deletions

View file

@ -33,7 +33,7 @@ ASCII2MAN = @echo "ERROR: rst2man from docutils command is not installed but is
endif endif
PYTHON=python PYTHON=python
GENERATE_CLI = $(PYTHON) docs/bin/generate_man.py GENERATE_CLI = hacking/build-ansible.py generate-man
SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")

View file

@ -1,74 +0,0 @@
#!/usr/bin/env python
import optparse
import os
import sys
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
from ansible.utils._build_helpers import update_file_if_different
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
def generate_parser():
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options]',
description='Generate module documentation from metadata',
)
p.add_option("-t", "--template-file", action="store", dest="template_file", default=DEFAULT_TEMPLATE_FILE, help="directory containing Jinja2 templates")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
(options, args) = p.parse_args()
return p
def fix_description(config_options):
'''some descriptions are strings, some are lists. workaround it...'''
for config_key in config_options:
description = config_options[config_key].get('description', [])
if isinstance(description, list):
desc_list = description
else:
desc_list = [description]
config_options[config_key]['description'] = desc_list
return config_options
def main(args):
parser = generate_parser()
(options, args) = parser.parse_args()
output_dir = os.path.abspath(options.output_dir)
template_file_full_path = os.path.abspath(options.template_file)
template_file = os.path.basename(template_file_full_path)
template_dir = os.path.dirname(os.path.abspath(template_file_full_path))
if options.docs:
with open(options.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
config_options = docs
config_options = fix_description(config_options)
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(template_file)
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
temp_vars = {'config_options': config_options}
data = to_bytes(template.render(temp_vars))
update_file_if_different(output_name, data)
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[:]))

View file

@ -1,84 +0,0 @@
#!/usr/bin/env python
import optparse
import re
from distutils.version import LooseVersion
import jinja2
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
from ansible.playbook import Play
from ansible.playbook.block import Block
from ansible.playbook.role import Role
from ansible.playbook.task import Task
from ansible.utils._build_helpers import update_file_if_different
template_file = 'playbooks_keywords.rst.j2'
oblist = {}
clist = []
class_list = [Play, Role, Block, Task]
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options]',
description='Generate playbook keyword documentation from code and descriptions',
)
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="../templates", help="directory containing Jinja2 templates")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
(options, args) = p.parse_args()
for aclass in class_list:
aobj = aclass()
name = type(aobj).__name__
if options.docs:
with open(options.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
# build ordered list to loop over and dict with attributes
clist.append(name)
oblist[name] = dict((x, aobj.__dict__['_attributes'][x]) for x in aobj.__dict__['_attributes'] if 'private' not in x or not x.private)
# pick up docs if they exist
for a in oblist[name]:
if a in docs:
oblist[name][a] = docs[a]
else:
# check if there is an alias, otherwise undocumented
alias = getattr(getattr(aobj, '_%s' % a), 'alias', None)
if alias and alias in docs:
oblist[name][alias] = docs[alias]
del oblist[name][a]
else:
oblist[name][a] = ' UNDOCUMENTED!! '
# loop is really with_ for users
if name == 'Task':
oblist[name]['with_<lookup_plugin>'] = 'The same as ``loop`` but magically adds the output of any lookup plugin to generate the item list.'
# local_action is implicit with action
if 'action' in oblist[name]:
oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``'
# remove unusable (used to be private?)
for nouse in ('loop_args', 'loop_with'):
if nouse in oblist[name]:
del oblist[name][nouse]
env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
template = env.get_template(template_file)
outputname = options.output_dir + template_file.replace('.j2', '')
tempvars = {'oblist': oblist, 'clist': clist}
keyword_page = template.render(tempvars)
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
keyword_page = re.sub(' +\n', '\n', keyword_page)
update_file_if_different(outputname, to_bytes(keyword_page))

View file

@ -1,10 +1,10 @@
OS := $(shell uname -s) OS := $(shell uname -s)
SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"): SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
FORMATTER=../bin/plugin_formatter.py PLUGIN_FORMATTER=../../hacking/build-ansible.py document-plugins
TESTING_FORMATTER=../bin/testing_formatter.sh TESTING_FORMATTER=../bin/testing_formatter.sh
DUMPER=../bin/dump_keywords.py KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords
CONFIG_DUMPER=../bin/dump_config.py CONFIG_DUMPER=../../hacking/build-ansible.py document-config
GENERATE_CLI=../bin/generate_man.py GENERATE_CLI=../../hacking/build-ansible.py generate-man
ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1) ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}') CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
else else
@ -79,24 +79,24 @@ clean:
.PHONY: docs clean .PHONY: docs clean
# TODO: make generate_man output dir cli option # TODO: make generate_man output dir cli option
cli: $(GENERATE_CLI) cli:
mkdir -p rst/cli mkdir -p rst/cli
PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py
keywords: $(FORMATTER) ../templates/playbooks_keywords.rst.j2 keywords: ../templates/playbooks_keywords.rst.j2
PYTHONPATH=../../lib $(DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml PYTHONPATH=../../lib $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
config: config: ../templates/config.rst.j2
PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml
modules: $(FORMATTER) ../templates/plugin.rst.j2 modules: ../templates/plugin.rst.j2
PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS) PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
plugins: $(FORMATTER) ../templates/plugin.rst.j2 plugins: ../templates/plugin.rst.j2
@echo "looping over doc plugins" @echo "looping over doc plugins"
for plugin in $(DOC_PLUGINS); \ for plugin in $(DOC_PLUGINS); \
do \ do \
PYTHONPATH=../../lib $(FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \ PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
done done
testing: testing:

View file

@ -5,3 +5,4 @@ rstcheck
sphinx sphinx
sphinx-notfound-page sphinx-notfound-page
Pygments >= 2.4.0 Pygments >= 2.4.0
straight.plugin # Needed for hacking/build-ansible.py which is the backend build script

View file

@ -60,7 +60,8 @@ If you make multiple changes to the documentation, or add more than a line to it
#. Test your changes for rST errors. #. Test your changes for rST errors.
#. Build the page, and preferably the entire documentation site, locally. #. Build the page, and preferably the entire documentation site, locally.
To work with documentation on your local machine, you need the following packages installed: To work with documentation on your local machine, you need to have python-3.5 or greater and the
following packages installed:
- gcc - gcc
- jinja2 - jinja2
@ -72,6 +73,7 @@ To work with documentation on your local machine, you need the following package
- six - six
- sphinx - sphinx
- sphinx-notfound-page - sphinx-notfound-page
- straight.plugin
.. note:: .. note::

View file

@ -19,15 +19,15 @@ These are the keywords available on common playbook objects. Keywords are one of
:local: :local:
:depth: 1 :depth: 1
{% for name in clist %} {% for name in playbook_class_names %}
{{ name }} {{ name }}
{{ '-' * name|length }} {{ '-' * name|length }}
.. glossary:: .. glossary::
{% for attribute in oblist[name]|sort %} {% for attribute in pb_keywords[name]|sort %}
{{ attribute }} {{ attribute }}
{{ oblist[name][attribute] |indent(8) }} {{ pb_keywords[name][attribute] |indent(8) }}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}

View file

@ -21,18 +21,17 @@ except ImportError:
argcomplete = None argcomplete = None
def set_sys_path(this_script=__file__): def build_lib_path(this_script=__file__):
"""Add path to the common librarydirectory to :attr:`sys.path`""" """Return path to the common build library directory"""
hacking_dir = os.path.dirname(this_script) hacking_dir = os.path.dirname(this_script)
libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library')) libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
if libdir not in sys.path: return libdir
sys.path.insert(0, libdir)
set_sys_path() sys.path.insert(0, build_lib_path())
from build_ansible import commands from build_ansible import commands, errors
def create_arg_parser(program_name): def create_arg_parser(program_name):
@ -63,13 +62,26 @@ def main():
argcomplete.autocomplete(arg_parser) argcomplete.autocomplete(arg_parser)
args = arg_parser.parse_args(sys.argv[1:]) args = arg_parser.parse_args(sys.argv[1:])
if args.command is None:
print('Please specify a subcommand to run')
sys.exit(1)
for subcommand in subcommands: for subcommand in subcommands:
if subcommand.name == args.command: if subcommand.name == args.command:
sys.exit(subcommand.main(args)) command = subcommand
break
else:
# Note: We should never trigger this because argparse should shield us from it
print('Error: {0} was not a recognized subcommand'.format(args.command))
sys.exit(1)
print('Error: Select a subcommand') try:
arg_parser.print_usage() retval = command.main(args)
except errors.DependencyError as e:
print(e)
sys.exit(2)
sys.exit(retval)
if __name__ == '__main__': if __name__ == '__main__':

View file

@ -0,0 +1,80 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import os.path
import pathlib
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
def fix_description(config_options):
'''some descriptions are strings, some are lists. workaround it...'''
for config_key in config_options:
description = config_options[config_key].get('description', [])
if isinstance(description, list):
desc_list = description
else:
desc_list = [description]
config_options[config_key]['description'] = desc_list
return config_options
class DocumentConfig(Command):
name = 'document-config'
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(cls.name, description='Generate module documentation from metadata')
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
default=DEFAULT_TEMPLATE_FILE,
help="Jinja2 template to use for the config")
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
default=DEFAULT_TEMPLATE_DIR,
help="directory containing Jinja2 templates")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
help="Output directory for rst files")
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
help="Source for attribute docs")
@staticmethod
def main(args):
output_dir = os.path.abspath(args.output_dir)
template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
template_file = os.path.basename(template_file_full_path)
template_dir = os.path.dirname(template_file_full_path)
if args.docs:
with open(args.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
config_options = docs
config_options = fix_description(config_options)
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(template_file)
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
temp_vars = {'config_options': config_options}
data = to_bytes(template.render(temp_vars))
update_file_if_different(output_name, data)
return 0

View file

@ -0,0 +1,125 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import importlib
import os.path
import pathlib
import re
from distutils.version import LooseVersion
import jinja2
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
DEFAULT_TEMPLATE_DIR = str(pathlib.Path(__file__).resolve().parents[4] / 'docs/templates')
TEMPLATE_FILE = 'playbooks_keywords.rst.j2'
PLAYBOOK_CLASS_NAMES = ['Play', 'Role', 'Block', 'Task']
def load_definitions(keyword_definitions_file):
docs = {}
with open(keyword_definitions_file) as f:
docs = yaml.safe_load(f)
return docs
def extract_keywords(keyword_definitions):
pb_keywords = {}
for pb_class_name in PLAYBOOK_CLASS_NAMES:
if pb_class_name == 'Play':
module_name = 'ansible.playbook'
else:
module_name = 'ansible.playbook.{0}'.format(pb_class_name.lower())
module = importlib.import_module(module_name)
playbook_class = getattr(module, pb_class_name, None)
if playbook_class is None:
raise ImportError("We weren't able to import the module {0}".format(module_name))
# Maintain order of the actual class names for our output
# Build up a mapping of playbook classes to the attributes that they hold
pb_keywords[pb_class_name] = {k: v for (k, v) in playbook_class._valid_attrs.items()
# Filter private attributes as they're not usable in playbooks
if not v.private}
# pick up definitions if they exist
for keyword in tuple(pb_keywords[pb_class_name]):
if keyword in keyword_definitions:
pb_keywords[pb_class_name][keyword] = keyword_definitions[keyword]
else:
# check if there is an alias, otherwise undocumented
alias = getattr(getattr(playbook_class, '_%s' % keyword), 'alias', None)
if alias and alias in keyword_definitions:
pb_keywords[pb_class_name][alias] = keyword_definitions[alias]
del pb_keywords[pb_class_name][keyword]
else:
pb_keywords[pb_class_name][keyword] = ' UNDOCUMENTED!! '
# loop is really with_ for users
if pb_class_name == 'Task':
pb_keywords[pb_class_name]['with_<lookup_plugin>'] = (
'The same as ``loop`` but magically adds the output of any lookup plugin to'
' generate the item list.')
# local_action is implicit with action
if 'action' in pb_keywords[pb_class_name]:
pb_keywords[pb_class_name]['local_action'] = ('Same as action but also implies'
' ``delegate_to: localhost``')
return pb_keywords
def generate_page(pb_keywords, template_dir):
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(TEMPLATE_FILE)
tempvars = {'pb_keywords': pb_keywords, 'playbook_class_names': PLAYBOOK_CLASS_NAMES}
keyword_page = template.render(tempvars)
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
keyword_page = re.sub(' +\n', '\n', keyword_page)
return keyword_page
class DocumentKeywords(Command):
name = 'document-keywords'
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(cls.name, description='Generate playbook keyword documentation from'
' code and descriptions')
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
default=DEFAULT_TEMPLATE_DIR,
help="directory containing Jinja2 templates")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
default='/tmp/', help="Output directory for rst files")
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
help="Source for attribute docs")
@staticmethod
def main(args):
if not args.docs:
print('Definitions for keywords must be specified via `--docs-source FILENAME`')
return 1
keyword_definitions = load_definitions(args.docs)
pb_keywords = extract_keywords(keyword_definitions)
keyword_page = generate_page(pb_keywords, args.template_dir)
outputname = os.path.join(args.output_dir, TEMPLATE_FILE.replace('.j2', ''))
update_file_if_different(outputname, to_bytes(keyword_page))
return 0

View file

@ -1,25 +1,27 @@
#!/usr/bin/env python # coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import argparse import argparse
import os import os.path
import pathlib
import sys import sys
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes from ansible.module_utils._text import to_bytes
from ansible.utils._build_helpers import update_file_if_different
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
def generate_parser(): DEFAULT_TEMPLATE_FILE = pathlib.Path(__file__).parents[4] / 'docs/templates/man.j2'
p = argparse.ArgumentParser(
description='Generate cli documentation from cli docstrings',
)
p.add_argument("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
p.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_argument("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
p.add_argument('args', help='CLI module(s)', metavar='module', nargs='*')
return p
# from https://www.python.org/dev/peps/pep-0257/ # from https://www.python.org/dev/peps/pep-0257/
@ -213,78 +215,89 @@ def opts_docs(cli_class_name, cli_module_name):
return docs return docs
if __name__ == '__main__': class GenerateMan(Command):
name = 'generate-man'
parser = generate_parser() @classmethod
def init_parser(cls, add_parser):
parser = add_parser(name=cls.name,
description='Generate cli documentation from cli docstrings')
options = parser.parse_args() parser.add_argument("-t", "--template-file", action="store", dest="template_file",
default=DEFAULT_TEMPLATE_FILE, help="path to jinja2 template")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
default='/tmp/', help="Output directory for rst files")
parser.add_argument("-f", "--output-format", action="store", dest="output_format",
default='man',
help="Output format for docs (the default 'man' or 'rst')")
parser.add_argument('cli_modules', help='CLI module name(s)', metavar='MODULE_NAME', nargs='*')
template_file = options.template_file @staticmethod
template_path = os.path.expanduser(template_file) def main(args):
template_dir = os.path.abspath(os.path.dirname(template_path)) template_file = args.template_file
template_basename = os.path.basename(template_file) template_path = os.path.expanduser(template_file)
template_dir = os.path.abspath(os.path.dirname(template_path))
template_basename = os.path.basename(template_file)
output_dir = os.path.abspath(options.output_dir) output_dir = os.path.abspath(args.output_dir)
output_format = options.output_format output_format = args.output_format
cli_modules = options.args cli_modules = args.cli_modules
# various cli parsing things checks sys.argv if the 'args' that are passed in are [] # various cli parsing things checks sys.argv if the 'args' that are passed in are []
# so just remove any args so the cli modules dont try to parse them resulting in warnings # so just remove any args so the cli modules dont try to parse them resulting in warnings
sys.argv = [sys.argv[0]] sys.argv = [sys.argv[0]]
# need to be in right dir
os.chdir(os.path.dirname(__file__))
allvars = {} allvars = {}
output = {} output = {}
cli_list = [] cli_list = []
cli_bin_name_list = [] cli_bin_name_list = []
# for binary in os.listdir('../../lib/ansible/cli'): # for binary in os.listdir('../../lib/ansible/cli'):
for cli_module_name in cli_modules: for cli_module_name in cli_modules:
binary = os.path.basename(os.path.expanduser(cli_module_name)) binary = os.path.basename(os.path.expanduser(cli_module_name))
if not binary.endswith('.py'): if not binary.endswith('.py'):
continue continue
elif binary == '__init__.py': elif binary == '__init__.py':
continue continue
cli_name = os.path.splitext(binary)[0] cli_name = os.path.splitext(binary)[0]
if cli_name == 'adhoc': if cli_name == 'adhoc':
cli_class_name = 'AdHocCLI' cli_class_name = 'AdHocCLI'
# myclass = 'AdHocCLI' # myclass = 'AdHocCLI'
output[cli_name] = 'ansible.1.rst.in' output[cli_name] = 'ansible.1.rst.in'
cli_bin_name = 'ansible' cli_bin_name = 'ansible'
else: else:
# myclass = "%sCLI" % libname.capitalize() # myclass = "%sCLI" % libname.capitalize()
cli_class_name = "%sCLI" % cli_name.capitalize() cli_class_name = "%sCLI" % cli_name.capitalize()
output[cli_name] = 'ansible-%s.1.rst.in' % cli_name output[cli_name] = 'ansible-%s.1.rst.in' % cli_name
cli_bin_name = 'ansible-%s' % cli_name cli_bin_name = 'ansible-%s' % cli_name
# FIXME: # FIXME:
allvars[cli_name] = opts_docs(cli_class_name, cli_name) allvars[cli_name] = opts_docs(cli_class_name, cli_name)
cli_bin_name_list.append(cli_bin_name) cli_bin_name_list.append(cli_bin_name)
cli_list = allvars.keys() cli_list = allvars.keys()
doc_name_formats = {'man': '%s.1.rst.in', doc_name_formats = {'man': '%s.1.rst.in',
'rst': '%s.rst'} 'rst': '%s.rst'}
for cli_name in cli_list: for cli_name in cli_list:
# template it! # template it!
env = Environment(loader=FileSystemLoader(template_dir)) env = Environment(loader=FileSystemLoader(template_dir))
template = env.get_template(template_basename) template = env.get_template(template_basename)
# add rest to vars # add rest to vars
tvars = allvars[cli_name] tvars = allvars[cli_name]
tvars['cli_list'] = cli_list tvars['cli_list'] = cli_list
tvars['cli_bin_name_list'] = cli_bin_name_list tvars['cli_bin_name_list'] = cli_bin_name_list
tvars['cli'] = cli_name tvars['cli'] = cli_name
if '-i' in tvars['options']: if '-i' in tvars['options']:
print('uses inventory') print('uses inventory')
manpage = template.render(tvars) manpage = template.render(tvars)
filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name']) filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
update_file_if_different(filename, to_bytes(manpage)) update_file_if_different(filename, to_bytes(manpage))

View file

@ -1,4 +1,3 @@
#!/usr/bin/env python
# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com> # Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others # Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
# Copyright: (c) 2017, Ansible Project # Copyright: (c) 2017, Ansible Project
@ -45,7 +44,10 @@ from ansible.module_utils.six import iteritems, string_types
from ansible.plugins.loader import fragment_loader from ansible.plugins.loader import fragment_loader
from ansible.utils import plugin_docs from ansible.utils import plugin_docs
from ansible.utils.display import Display from ansible.utils.display import Display
from ansible.utils._build_helpers import update_file_if_different
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
##################################################################################### #####################################################################################
@ -363,29 +365,6 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
return module_info, categories return module_info, categories
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2',
description='Generate module documentation from metadata',
)
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)")
p.add_option("-T", "--template-dir", action="append", dest="template_dir", help="directory containing Jinja2 templates")
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None,
help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.")
p.add_option('-V', action='version', help='Show version number and exit')
p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)")
return p
def jinja2_environment(template_dir, typ, plugin_type): def jinja2_environment(template_dir, typ, plugin_type):
env = Environment(loader=FileSystemLoader(template_dir), env = Environment(loader=FileSystemLoader(template_dir),
@ -734,82 +713,106 @@ def validate_options(options):
sys.exit("--template-dir must be specified") sys.exit("--template-dir must be specified")
def main(): class DocumentPlugins(Command):
name = 'document-plugins'
# INIT @classmethod
p = generate_parser() def init_parser(cls, add_parser):
(options, args) = p.parse_args() parser = add_parser(cls.name, description='Generate module documentation from metadata')
if not options.template_dir:
options.template_dir = ["hacking/templates"]
validate_options(options)
display.verbosity = options.verbosity
plugin_type = options.plugin_type
display.display("Evaluating %s files..." % plugin_type) parser.add_argument("-A", "--ansible-version", action="store", dest="ansible_version",
default="unknown", help="Ansible version number")
parser.add_argument("-M", "--module-dir", action="store", dest="module_dir",
default=MODULEDIR, help="Ansible library path")
parser.add_argument("-P", "--plugin-type", action="store", dest="plugin_type",
default='module', help="The type of plugin (module, lookup, etc)")
parser.add_argument("-T", "--template-dir", action="append", dest="template_dir",
help="directory containing Jinja2 templates")
parser.add_argument("-t", "--type", action='store', dest='type', choices=['rst'],
default='rst', help="Document type")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default=None,
help="Output directory for module files")
parser.add_argument("-I", "--includes-file", action="store", dest="includes_file",
default=None, help="Create a file containing list of processed modules")
parser.add_argument("-l", "--limit-to-modules", '--limit-to', action="store",
dest="limit_to", default=None, help="Limit building module documentation"
" to comma-separated list of plugins. Specify non-existing plugin name"
" for no plugins.")
parser.add_argument('-V', action='version', help='Show version number and exit')
parser.add_argument('-v', '--verbose', dest='verbosity', default=0, action="count",
help="verbose mode (increase number of 'v's for more)")
# prep templating @staticmethod
templates = jinja2_environment(options.template_dir, options.type, plugin_type) def main(args):
if not args.template_dir:
args.template_dir = ["hacking/templates"]
validate_options(args)
display.verbosity = args.verbosity
plugin_type = args.plugin_type
# set file/directory structure display.display("Evaluating %s files..." % plugin_type)
if plugin_type == 'module':
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
outputname = '%s_' + '%s.rst' % plugin_type
output_dir = options.output_dir
else:
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
outputname = '%s.rst'
output_dir = '%s/plugins/%s' % (options.output_dir, plugin_type)
display.vv('output name: %s' % outputname) # prep templating
display.vv('output dir: %s' % output_dir) templates = jinja2_environment(args.template_dir, args.type, plugin_type)
# Convert passed-in limit_to to None or list of modules. # set file/directory structure
if options.limit_to is not None: if plugin_type == 'module':
options.limit_to = [s.lower() for s in options.limit_to.split(",")] # trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
outputname = '%s_' + '%s.rst' % plugin_type
output_dir = args.output_dir
else:
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
outputname = '%s.rst'
output_dir = '%s/plugins/%s' % (args.output_dir, plugin_type)
plugin_info, categories = get_plugin_info(options.module_dir, limit_to=options.limit_to, verbose=(options.verbosity > 0)) display.vv('output name: %s' % outputname)
display.vv('output dir: %s' % output_dir)
categories['all'] = {'_modules': plugin_info.keys()} # Convert passed-in limit_to to None or list of modules.
if args.limit_to is not None:
args.limit_to = [s.lower() for s in args.limit_to.split(",")]
if display.verbosity >= 3: plugin_info, categories = get_plugin_info(args.module_dir, limit_to=args.limit_to, verbose=(args.verbosity > 0))
display.vvv(pp.pformat(categories))
if display.verbosity >= 5:
display.vvvvv(pp.pformat(plugin_info))
# Transform the data categories['all'] = {'_modules': plugin_info.keys()}
if options.type == 'rst':
display.v('Generating rst')
for key, record in plugin_info.items():
display.vv(key)
if display.verbosity >= 5:
display.vvvvv(pp.pformat(('record', record)))
if record.get('doc', None):
short_desc = record['doc']['short_description'].rstrip('.')
if short_desc is None:
display.warning('short_description for %s is None' % key)
short_desc = ''
record['doc']['short_description'] = rst_ify(short_desc)
if plugin_type == 'module': if display.verbosity >= 3:
display.v('Generating Categories') display.vvv(pp.pformat(categories))
# Write module master category list if display.verbosity >= 5:
category_list_text = templates['category_list'].render(categories=sorted(categories.keys())) display.vvvvv(pp.pformat(plugin_info))
category_index_name = '%ss_by_category.rst' % plugin_type
write_data(category_list_text, output_dir, category_index_name)
# Render all the individual plugin pages # Transform the data
display.v('Generating plugin pages') if args.type == 'rst':
process_plugins(plugin_info, templates, outputname, output_dir, options.ansible_version, plugin_type) display.v('Generating rst')
for key, record in plugin_info.items():
display.vv(key)
if display.verbosity >= 5:
display.vvvvv(pp.pformat(('record', record)))
if record.get('doc', None):
short_desc = record['doc']['short_description'].rstrip('.')
if short_desc is None:
display.warning('short_description for %s is None' % key)
short_desc = ''
record['doc']['short_description'] = rst_ify(short_desc)
# Render all the categories for modules if plugin_type == 'module':
if plugin_type == 'module': display.v('Generating Categories')
display.v('Generating Category lists') # Write module master category list
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type) category_index_name = '%ss_by_category.rst' % plugin_type
write_data(category_list_text, output_dir, category_index_name)
# Render all the individual plugin pages
display.v('Generating plugin pages')
process_plugins(plugin_info, templates, outputname, output_dir, args.ansible_version, plugin_type)
# Render all the categories for modules # Render all the categories for modules
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type) if plugin_type == 'module':
display.v('Generating Category lists')
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
# Render all the categories for modules
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
if __name__ == '__main__': return 0
main()

View file

@ -121,7 +121,7 @@ def generate_porting_guide(version):
def write_guide(version, guide_content): def write_guide(version, guide_content):
filename = f'porting_guide_{version}.rst' filename = 'porting_guide_{0}.rst'.format(version)
with open(filename, 'w') as out_file: with open(filename, 'w') as out_file:
out_file.write(guide_content) out_file.write(guide_content)

View file

@ -7,148 +7,13 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import argparse
import asyncio
import datetime
import hashlib
import os.path
import sys import sys
from collections import UserString from collections import UserString
from distutils.version import LooseVersion from distutils.version import LooseVersion
import aiohttp
from jinja2 import Environment, DictLoader
# Pylint doesn't understand Python3 namespace modules. # Pylint doesn't understand Python3 namespace modules.
from ..commands import Command # pylint: disable=relative-beyond-top-level from ..commands import Command # pylint: disable=relative-beyond-top-level
from .. import errors # pylint: disable=relative-beyond-top-level
# pylint: disable=
VERSION_FRAGMENT = """
{%- if versions | length > 1 %}
{% for version in versions %}
{% if loop.last %}and {{ version }}{% else %}
{% if versions | length == 2 %}{{ version }} {% else %}{{ version }}, {% endif -%}
{% endif -%}
{% endfor -%}
{%- else %}{{ versions[0] }}{% endif -%}
"""
LONG_TEMPLATE = """
{% set plural = False if versions | length == 1 else True %}
{% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
Subject: New Ansible release{% if plural %}s{% endif %} {{ version_str }}
{% filter wordwrap %}
Hi all- we're happy to announce that the general release of Ansible {{ version_str }}{% if plural %} are{%- else %} is{%- endif %} now available!
{% endfilter %}
How do you get it?
------------------
{% for version in versions %}
$ pip install ansible=={{ version }} --user
{% if not loop.last %}
or
{% endif %}
{% endfor %}
The tar.gz of the release{% if plural %}s{% endif %} can be found here:
{% for version in versions %}
* {{ version }}
https://releases.ansible.com/ansible/ansible-{{ version }}.tar.gz
SHA256: {{ hashes[version] }}
{% endfor %}
What's new in {{ version_str }}
{{ '-' * (14 + version_str | length) }}
{% filter wordwrap %}
{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
{% endfilter %}
{% for version in versions %}
* {{ version }}
https://github.com/ansible/ansible/blob/stable-{{ version.split('.')[:2] | join('.') }}/changelogs/CHANGELOG-v{{ version.split('.')[:2] | join('.') }}.rst
{% endfor %}
What's the schedule for future maintenance releases?
----------------------------------------------------
{% filter wordwrap %}
Future maintenance releases will occur approximately every 3 weeks. So expect the next one around {{ next_release.strftime('%Y-%m-%d') }}.
{% endfilter %}
Porting Help
------------
{% filter wordwrap %}
We've published a porting guide at
https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_{{ latest_ver.split('.')[:2] | join('.') }}.html to help migrate your content to {{ latest_ver.split('.')[:2] | join('.') }}.
{% endfilter %}
{% filter wordwrap %}
If you discover any errors or if any of your working playbooks break when you upgrade to {{ latest_ver }}, please use the following link to report the regression:
{% endfilter %}
https://github.com/ansible/ansible/issues/new/choose
{% filter wordwrap %}
In your issue, be sure to mention the Ansible version that works and the one that doesn't.
{% endfilter %}
Thanks!
-{{ name }}
""" # noqa for E501 (line length).
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
# proper wrapping to occur
SHORT_TEMPLATE = """
{% set plural = False if versions | length == 1 else True %}
@ansible
{{ version_str }}
{% if plural %}
have
{% else %}
has
{% endif %}
been released! Get
{% if plural %}
them
{% else %}
it
{% endif %}
on PyPI: pip install ansible=={{ (versions|sort(attribute='ver_obj'))[-1] }},
https://releases.ansible.com/ansible/, the Ansible PPA on Launchpad, or GitHub. Happy automating!
""" # noqa for E501 (line length).
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
# proper wrapping to occur
JINJA_ENV = Environment(
loader=DictLoader({'long': LONG_TEMPLATE,
'short': SHORT_TEMPLATE,
'version_string': VERSION_FRAGMENT,
}),
extensions=['jinja2.ext.i18n'],
trim_blocks=True,
lstrip_blocks=True,
)
class VersionStr(UserString): class VersionStr(UserString):
@ -167,108 +32,6 @@ def transform_args(args):
return args return args
async def calculate_hash_from_tarball(session, version):
tar_url = f'https://releases.ansible.com/ansible/ansible-{version}.tar.gz'
tar_task = asyncio.create_task(session.get(tar_url))
tar_response = await tar_task
tar_hash = hashlib.sha256()
while True:
chunk = await tar_response.content.read(1024)
if not chunk:
break
tar_hash.update(chunk)
return tar_hash.hexdigest()
async def parse_hash_from_file(session, version):
filename = f'ansible-{version}.tar.gz'
hash_url = f'https://releases.ansible.com/ansible/{filename}.sha'
hash_task = asyncio.create_task(session.get(hash_url))
hash_response = await hash_task
hash_content = await hash_response.read()
precreated_hash, precreated_filename = hash_content.split(None, 1)
if filename != precreated_filename.strip().decode('utf-8'):
raise ValueError(f'Hash file contains hash for a different file: {precreated_filename}')
return precreated_hash.decode('utf-8')
async def get_hash(session, version):
calculated_hash = await calculate_hash_from_tarball(session, version)
precreated_hash = await parse_hash_from_file(session, version)
if calculated_hash != precreated_hash:
raise ValueError(f'Hash in file ansible-{version}.tar.gz.sha {precreated_hash} does not'
f' match hash of tarball {calculated_hash}')
return calculated_hash
async def get_hashes(versions):
hashes = {}
requestors = {}
async with aiohttp.ClientSession() as aio_session:
for version in versions:
requestors[version] = asyncio.create_task(get_hash(aio_session, version))
for version, request in requestors.items():
await request
hashes[version] = request.result()
return hashes
def next_release_date(weeks=3):
days_in_the_future = weeks * 7
today = datetime.datetime.now()
numeric_today = today.weekday()
# We release on Thursdays
if numeric_today == 3:
# 3 is Thursday
pass
elif numeric_today == 4:
# If this is Friday, we can adjust back to Thursday for the next release
today -= datetime.timedelta(days=1)
elif numeric_today < 3:
# Otherwise, slide forward to Thursday
today += datetime.timedelta(days=(3 - numeric_today))
else:
# slightly different formula if it's past Thursday this week. We need to go forward to
# Thursday of next week
today += datetime.timedelta(days=(10 - numeric_today))
next_release = today + datetime.timedelta(days=days_in_the_future)
return next_release
def generate_long_message(versions, name):
hashes = asyncio.run(get_hashes(versions))
version_template = JINJA_ENV.get_template('version_string')
version_str = version_template.render(versions=versions).strip()
next_release = next_release_date()
template = JINJA_ENV.get_template('long')
message = template.render(versions=versions, version_str=version_str,
name=name, hashes=hashes, next_release=next_release)
return message
def generate_short_message(versions):
version_template = JINJA_ENV.get_template('version_string')
version_str = version_template.render(versions=versions).strip()
template = JINJA_ENV.get_template('short')
message = template.render(versions=versions, version_str=version_str)
message = ' '.join(message.split()) + '\n'
return message
def write_message(filename, message): def write_message(filename, message):
if filename != '-': if filename != '-':
with open(filename, 'w') as out_file: with open(filename, 'w') as out_file:
@ -294,12 +57,21 @@ class ReleaseAnnouncementCommand(Command):
parser.add_argument("--twitter-out", type=str, default="-", parser.add_argument("--twitter-out", type=str, default="-",
help="Filename to place the twitter announcement into") help="Filename to place the twitter announcement into")
@staticmethod @classmethod
def main(args): def main(cls, args):
if sys.version_info < (3, 6):
raise errors.DependencyError('The {0} subcommand needs Python-3.6+'
' to run'.format(cls.name))
# Import here because these functions are invalid on Python-3.5 and the command plugins and
# init_parser() method need to be compatible with Python-3.4+ for now.
# Pylint doesn't understand Python3 namespace modules.
from .. announce import create_short_message, create_long_message # pylint: disable=relative-beyond-top-level
args = transform_args(args) args = transform_args(args)
twitter_message = generate_short_message(args.versions) twitter_message = create_short_message(args.versions)
email_message = generate_long_message(args.versions, args.name) email_message = create_long_message(args.versions, args.name)
write_message(args.twitter_out, twitter_message) write_message(args.twitter_out, twitter_message)
write_message(args.email_out, email_message) write_message(args.email_out, email_message)

View file

@ -0,0 +1,12 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
class DependencyError(Exception):
"""Used when a dependency is unmet"""
pass

View file

@ -52,8 +52,9 @@ class Attribute:
:kwarg isa: The type of the attribute. Allowable values are a string :kwarg isa: The type of the attribute. Allowable values are a string
representation of any yaml basic datatype, python class, or percent. representation of any yaml basic datatype, python class, or percent.
(Enforced at post-validation time). (Enforced at post-validation time).
:kwarg private: Hides the attribute from being documented. :kwarg private: Not used at runtime. The docs playbook keyword dumper uses it to determine
TODO: it should also should prevent it from being user settable that a keyword should not be documented. mpdehaan had plans to remove attributes marked
private from the ds so they would not have been available at all.
:kwarg default: Default value if unspecified in the YAML document. :kwarg default: Default value if unspecified in the YAML document.
:kwarg required: Whether or not the YAML document must contain this field. :kwarg required: Whether or not the YAML document must contain this field.
If the attribute is None when post-validated, an error will be raised. If the attribute is None when post-validated, an error will be raised.

View file

@ -1,3 +1,11 @@
# The following are only run by release engineers who can be asked to have newer Python3 on their systems # The following are only run by release engineers who can be asked to have newer Python3 on their systems
hacking/build_library/build_ansible/command_plugins/porting_guide.py hacking/build_library/build_ansible/command_plugins/porting_guide.py
hacking/build_library/build_ansible/command_plugins/release_announcement.py hacking/build_library/build_ansible/command_plugins/release_announcement.py
# The following are used to build docs. Since we explicitly say that the controller won't run on
# Python-2.6 (docs are built controller-side) and EPEL-6, the only LTS platform with Python-2.6,
# doesn't have a new enough sphinx to build docs, do not test these under Python-2.6
hacking/build_library/build_ansible/command_plugins/dump_config.py
hacking/build_library/build_ansible/command_plugins/dump_keywords.py
hacking/build_library/build_ansible/command_plugins/generate_man.py
hacking/build_library/build_ansible/command_plugins/plugin_formatter.py