Move common build code from _build_helpers (#55986)
We have some common code used by several docs scripts. Migrate that into the build-only shared code repository. * Move lib/ansible/utils/_build_helpers.py to the directory for common build code * Migrate docs/bin/dump_config.py to a build-ansible subcommand * Migrate dump_keywords to the build-ansible framework * Make the script more maintainable by using functions and good variable names * Port to Python3 idioms * Fix bug so that private attributes will be undocumented * Move generate_man to a build-ansible subcommand * Port plugin_formatter to a build-ansible subcommand * Rework command_plugins so that docs scripts can target Python-3.4+ and releng-only subcommands can use more recent versions of Python. The architecture is now that command_plugins/* need to be importable on Python-3.4. The init_parsers() method needs to run on Python-3.4. But the main() method can utilize features of more recent Python as long as it fits within those parameters. * Update docs build requirements Port the plugin_formatter to build-ansible framework
This commit is contained in:
parent
65e0f37fc0
commit
019d078a5a
18 changed files with 457 additions and 586 deletions
2
Makefile
2
Makefile
|
@ -33,7 +33,7 @@ ASCII2MAN = @echo "ERROR: rst2man from docutils command is not installed but is
|
||||||
endif
|
endif
|
||||||
|
|
||||||
PYTHON=python
|
PYTHON=python
|
||||||
GENERATE_CLI = $(PYTHON) docs/bin/generate_man.py
|
GENERATE_CLI = hacking/build-ansible.py generate-man
|
||||||
|
|
||||||
SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
|
SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
|
||||||
|
|
||||||
|
|
|
@ -1,74 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import optparse
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
from jinja2 import Environment, FileSystemLoader
|
|
||||||
from ansible.module_utils._text import to_bytes
|
|
||||||
from ansible.utils._build_helpers import update_file_if_different
|
|
||||||
|
|
||||||
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
|
|
||||||
|
|
||||||
|
|
||||||
def generate_parser():
|
|
||||||
p = optparse.OptionParser(
|
|
||||||
version='%prog 1.0',
|
|
||||||
usage='usage: %prog [options]',
|
|
||||||
description='Generate module documentation from metadata',
|
|
||||||
)
|
|
||||||
p.add_option("-t", "--template-file", action="store", dest="template_file", default=DEFAULT_TEMPLATE_FILE, help="directory containing Jinja2 templates")
|
|
||||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
|
|
||||||
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
|
|
||||||
|
|
||||||
(options, args) = p.parse_args()
|
|
||||||
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def fix_description(config_options):
|
|
||||||
'''some descriptions are strings, some are lists. workaround it...'''
|
|
||||||
|
|
||||||
for config_key in config_options:
|
|
||||||
description = config_options[config_key].get('description', [])
|
|
||||||
if isinstance(description, list):
|
|
||||||
desc_list = description
|
|
||||||
else:
|
|
||||||
desc_list = [description]
|
|
||||||
config_options[config_key]['description'] = desc_list
|
|
||||||
return config_options
|
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
|
||||||
|
|
||||||
parser = generate_parser()
|
|
||||||
(options, args) = parser.parse_args()
|
|
||||||
|
|
||||||
output_dir = os.path.abspath(options.output_dir)
|
|
||||||
template_file_full_path = os.path.abspath(options.template_file)
|
|
||||||
template_file = os.path.basename(template_file_full_path)
|
|
||||||
template_dir = os.path.dirname(os.path.abspath(template_file_full_path))
|
|
||||||
|
|
||||||
if options.docs:
|
|
||||||
with open(options.docs) as f:
|
|
||||||
docs = yaml.safe_load(f)
|
|
||||||
else:
|
|
||||||
docs = {}
|
|
||||||
|
|
||||||
config_options = docs
|
|
||||||
config_options = fix_description(config_options)
|
|
||||||
|
|
||||||
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
|
||||||
template = env.get_template(template_file)
|
|
||||||
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
|
|
||||||
temp_vars = {'config_options': config_options}
|
|
||||||
|
|
||||||
data = to_bytes(template.render(temp_vars))
|
|
||||||
update_file_if_different(output_name, data)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main(sys.argv[:]))
|
|
|
@ -1,84 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import optparse
|
|
||||||
import re
|
|
||||||
from distutils.version import LooseVersion
|
|
||||||
|
|
||||||
import jinja2
|
|
||||||
import yaml
|
|
||||||
from jinja2 import Environment, FileSystemLoader
|
|
||||||
|
|
||||||
from ansible.module_utils._text import to_bytes
|
|
||||||
from ansible.playbook import Play
|
|
||||||
from ansible.playbook.block import Block
|
|
||||||
from ansible.playbook.role import Role
|
|
||||||
from ansible.playbook.task import Task
|
|
||||||
from ansible.utils._build_helpers import update_file_if_different
|
|
||||||
|
|
||||||
template_file = 'playbooks_keywords.rst.j2'
|
|
||||||
oblist = {}
|
|
||||||
clist = []
|
|
||||||
class_list = [Play, Role, Block, Task]
|
|
||||||
|
|
||||||
p = optparse.OptionParser(
|
|
||||||
version='%prog 1.0',
|
|
||||||
usage='usage: %prog [options]',
|
|
||||||
description='Generate playbook keyword documentation from code and descriptions',
|
|
||||||
)
|
|
||||||
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="../templates", help="directory containing Jinja2 templates")
|
|
||||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
|
|
||||||
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
|
|
||||||
|
|
||||||
(options, args) = p.parse_args()
|
|
||||||
|
|
||||||
for aclass in class_list:
|
|
||||||
aobj = aclass()
|
|
||||||
name = type(aobj).__name__
|
|
||||||
|
|
||||||
if options.docs:
|
|
||||||
with open(options.docs) as f:
|
|
||||||
docs = yaml.safe_load(f)
|
|
||||||
else:
|
|
||||||
docs = {}
|
|
||||||
|
|
||||||
# build ordered list to loop over and dict with attributes
|
|
||||||
clist.append(name)
|
|
||||||
oblist[name] = dict((x, aobj.__dict__['_attributes'][x]) for x in aobj.__dict__['_attributes'] if 'private' not in x or not x.private)
|
|
||||||
|
|
||||||
# pick up docs if they exist
|
|
||||||
for a in oblist[name]:
|
|
||||||
if a in docs:
|
|
||||||
oblist[name][a] = docs[a]
|
|
||||||
else:
|
|
||||||
# check if there is an alias, otherwise undocumented
|
|
||||||
alias = getattr(getattr(aobj, '_%s' % a), 'alias', None)
|
|
||||||
if alias and alias in docs:
|
|
||||||
oblist[name][alias] = docs[alias]
|
|
||||||
del oblist[name][a]
|
|
||||||
else:
|
|
||||||
oblist[name][a] = ' UNDOCUMENTED!! '
|
|
||||||
|
|
||||||
# loop is really with_ for users
|
|
||||||
if name == 'Task':
|
|
||||||
oblist[name]['with_<lookup_plugin>'] = 'The same as ``loop`` but magically adds the output of any lookup plugin to generate the item list.'
|
|
||||||
|
|
||||||
# local_action is implicit with action
|
|
||||||
if 'action' in oblist[name]:
|
|
||||||
oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``'
|
|
||||||
|
|
||||||
# remove unusable (used to be private?)
|
|
||||||
for nouse in ('loop_args', 'loop_with'):
|
|
||||||
if nouse in oblist[name]:
|
|
||||||
del oblist[name][nouse]
|
|
||||||
|
|
||||||
env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
|
|
||||||
template = env.get_template(template_file)
|
|
||||||
outputname = options.output_dir + template_file.replace('.j2', '')
|
|
||||||
tempvars = {'oblist': oblist, 'clist': clist}
|
|
||||||
|
|
||||||
keyword_page = template.render(tempvars)
|
|
||||||
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
|
|
||||||
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
|
|
||||||
keyword_page = re.sub(' +\n', '\n', keyword_page)
|
|
||||||
|
|
||||||
update_file_if_different(outputname, to_bytes(keyword_page))
|
|
|
@ -1,10 +1,10 @@
|
||||||
OS := $(shell uname -s)
|
OS := $(shell uname -s)
|
||||||
SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
|
SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
|
||||||
FORMATTER=../bin/plugin_formatter.py
|
PLUGIN_FORMATTER=../../hacking/build-ansible.py document-plugins
|
||||||
TESTING_FORMATTER=../bin/testing_formatter.sh
|
TESTING_FORMATTER=../bin/testing_formatter.sh
|
||||||
DUMPER=../bin/dump_keywords.py
|
KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords
|
||||||
CONFIG_DUMPER=../bin/dump_config.py
|
CONFIG_DUMPER=../../hacking/build-ansible.py document-config
|
||||||
GENERATE_CLI=../bin/generate_man.py
|
GENERATE_CLI=../../hacking/build-ansible.py generate-man
|
||||||
ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
|
ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
|
||||||
CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
|
CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
|
||||||
else
|
else
|
||||||
|
@ -79,24 +79,24 @@ clean:
|
||||||
.PHONY: docs clean
|
.PHONY: docs clean
|
||||||
|
|
||||||
# TODO: make generate_man output dir cli option
|
# TODO: make generate_man output dir cli option
|
||||||
cli: $(GENERATE_CLI)
|
cli:
|
||||||
mkdir -p rst/cli
|
mkdir -p rst/cli
|
||||||
PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py
|
PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py
|
||||||
|
|
||||||
keywords: $(FORMATTER) ../templates/playbooks_keywords.rst.j2
|
keywords: ../templates/playbooks_keywords.rst.j2
|
||||||
PYTHONPATH=../../lib $(DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
|
PYTHONPATH=../../lib $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
|
||||||
|
|
||||||
config:
|
config: ../templates/config.rst.j2
|
||||||
PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml
|
PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml
|
||||||
|
|
||||||
modules: $(FORMATTER) ../templates/plugin.rst.j2
|
modules: ../templates/plugin.rst.j2
|
||||||
PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
|
PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
|
||||||
|
|
||||||
plugins: $(FORMATTER) ../templates/plugin.rst.j2
|
plugins: ../templates/plugin.rst.j2
|
||||||
@echo "looping over doc plugins"
|
@echo "looping over doc plugins"
|
||||||
for plugin in $(DOC_PLUGINS); \
|
for plugin in $(DOC_PLUGINS); \
|
||||||
do \
|
do \
|
||||||
PYTHONPATH=../../lib $(FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
|
PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
|
||||||
done
|
done
|
||||||
|
|
||||||
testing:
|
testing:
|
||||||
|
|
|
@ -5,3 +5,4 @@ rstcheck
|
||||||
sphinx
|
sphinx
|
||||||
sphinx-notfound-page
|
sphinx-notfound-page
|
||||||
Pygments >= 2.4.0
|
Pygments >= 2.4.0
|
||||||
|
straight.plugin # Needed for hacking/build-ansible.py which is the backend build script
|
||||||
|
|
|
@ -60,7 +60,8 @@ If you make multiple changes to the documentation, or add more than a line to it
|
||||||
#. Test your changes for rST errors.
|
#. Test your changes for rST errors.
|
||||||
#. Build the page, and preferably the entire documentation site, locally.
|
#. Build the page, and preferably the entire documentation site, locally.
|
||||||
|
|
||||||
To work with documentation on your local machine, you need the following packages installed:
|
To work with documentation on your local machine, you need to have python-3.5 or greater and the
|
||||||
|
following packages installed:
|
||||||
|
|
||||||
- gcc
|
- gcc
|
||||||
- jinja2
|
- jinja2
|
||||||
|
@ -72,6 +73,7 @@ To work with documentation on your local machine, you need the following package
|
||||||
- six
|
- six
|
||||||
- sphinx
|
- sphinx
|
||||||
- sphinx-notfound-page
|
- sphinx-notfound-page
|
||||||
|
- straight.plugin
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|
||||||
|
|
6
docs/templates/playbooks_keywords.rst.j2
vendored
6
docs/templates/playbooks_keywords.rst.j2
vendored
|
@ -19,15 +19,15 @@ These are the keywords available on common playbook objects. Keywords are one of
|
||||||
:local:
|
:local:
|
||||||
:depth: 1
|
:depth: 1
|
||||||
|
|
||||||
{% for name in clist %}
|
{% for name in playbook_class_names %}
|
||||||
|
|
||||||
{{ name }}
|
{{ name }}
|
||||||
{{ '-' * name|length }}
|
{{ '-' * name|length }}
|
||||||
.. glossary::
|
.. glossary::
|
||||||
|
|
||||||
{% for attribute in oblist[name]|sort %}
|
{% for attribute in pb_keywords[name]|sort %}
|
||||||
{{ attribute }}
|
{{ attribute }}
|
||||||
{{ oblist[name][attribute] |indent(8) }}
|
{{ pb_keywords[name][attribute] |indent(8) }}
|
||||||
|
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
|
@ -21,18 +21,17 @@ except ImportError:
|
||||||
argcomplete = None
|
argcomplete = None
|
||||||
|
|
||||||
|
|
||||||
def set_sys_path(this_script=__file__):
|
def build_lib_path(this_script=__file__):
|
||||||
"""Add path to the common librarydirectory to :attr:`sys.path`"""
|
"""Return path to the common build library directory"""
|
||||||
hacking_dir = os.path.dirname(this_script)
|
hacking_dir = os.path.dirname(this_script)
|
||||||
libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
|
libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
|
||||||
|
|
||||||
if libdir not in sys.path:
|
return libdir
|
||||||
sys.path.insert(0, libdir)
|
|
||||||
|
|
||||||
|
|
||||||
set_sys_path()
|
sys.path.insert(0, build_lib_path())
|
||||||
|
|
||||||
from build_ansible import commands
|
from build_ansible import commands, errors
|
||||||
|
|
||||||
|
|
||||||
def create_arg_parser(program_name):
|
def create_arg_parser(program_name):
|
||||||
|
@ -63,13 +62,26 @@ def main():
|
||||||
argcomplete.autocomplete(arg_parser)
|
argcomplete.autocomplete(arg_parser)
|
||||||
|
|
||||||
args = arg_parser.parse_args(sys.argv[1:])
|
args = arg_parser.parse_args(sys.argv[1:])
|
||||||
|
if args.command is None:
|
||||||
|
print('Please specify a subcommand to run')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
for subcommand in subcommands:
|
for subcommand in subcommands:
|
||||||
if subcommand.name == args.command:
|
if subcommand.name == args.command:
|
||||||
sys.exit(subcommand.main(args))
|
command = subcommand
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# Note: We should never trigger this because argparse should shield us from it
|
||||||
|
print('Error: {0} was not a recognized subcommand'.format(args.command))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
print('Error: Select a subcommand')
|
try:
|
||||||
arg_parser.print_usage()
|
retval = command.main(args)
|
||||||
|
except errors.DependencyError as e:
|
||||||
|
print(e)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
sys.exit(retval)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -0,0 +1,80 @@
|
||||||
|
# coding: utf-8
|
||||||
|
# Copyright: (c) 2019, Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import os
|
||||||
|
import os.path
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
from ansible.module_utils._text import to_bytes
|
||||||
|
|
||||||
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
|
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||||
|
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
|
||||||
|
DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
|
||||||
|
|
||||||
|
|
||||||
|
def fix_description(config_options):
|
||||||
|
'''some descriptions are strings, some are lists. workaround it...'''
|
||||||
|
|
||||||
|
for config_key in config_options:
|
||||||
|
description = config_options[config_key].get('description', [])
|
||||||
|
if isinstance(description, list):
|
||||||
|
desc_list = description
|
||||||
|
else:
|
||||||
|
desc_list = [description]
|
||||||
|
config_options[config_key]['description'] = desc_list
|
||||||
|
return config_options
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentConfig(Command):
|
||||||
|
name = 'document-config'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def init_parser(cls, add_parser):
|
||||||
|
parser = add_parser(cls.name, description='Generate module documentation from metadata')
|
||||||
|
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
|
||||||
|
default=DEFAULT_TEMPLATE_FILE,
|
||||||
|
help="Jinja2 template to use for the config")
|
||||||
|
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
|
||||||
|
default=DEFAULT_TEMPLATE_DIR,
|
||||||
|
help="directory containing Jinja2 templates")
|
||||||
|
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
|
||||||
|
help="Output directory for rst files")
|
||||||
|
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
|
||||||
|
help="Source for attribute docs")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def main(args):
|
||||||
|
output_dir = os.path.abspath(args.output_dir)
|
||||||
|
template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
|
||||||
|
template_file = os.path.basename(template_file_full_path)
|
||||||
|
template_dir = os.path.dirname(template_file_full_path)
|
||||||
|
|
||||||
|
if args.docs:
|
||||||
|
with open(args.docs) as f:
|
||||||
|
docs = yaml.safe_load(f)
|
||||||
|
else:
|
||||||
|
docs = {}
|
||||||
|
|
||||||
|
config_options = docs
|
||||||
|
config_options = fix_description(config_options)
|
||||||
|
|
||||||
|
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
||||||
|
template = env.get_template(template_file)
|
||||||
|
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
|
||||||
|
temp_vars = {'config_options': config_options}
|
||||||
|
|
||||||
|
data = to_bytes(template.render(temp_vars))
|
||||||
|
update_file_if_different(output_name, data)
|
||||||
|
|
||||||
|
return 0
|
|
@ -0,0 +1,125 @@
|
||||||
|
# coding: utf-8
|
||||||
|
# Copyright: (c) 2019, Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import os.path
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
from distutils.version import LooseVersion
|
||||||
|
|
||||||
|
import jinja2
|
||||||
|
import yaml
|
||||||
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
|
||||||
|
from ansible.module_utils._text import to_bytes
|
||||||
|
|
||||||
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
|
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||||
|
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_TEMPLATE_DIR = str(pathlib.Path(__file__).resolve().parents[4] / 'docs/templates')
|
||||||
|
TEMPLATE_FILE = 'playbooks_keywords.rst.j2'
|
||||||
|
PLAYBOOK_CLASS_NAMES = ['Play', 'Role', 'Block', 'Task']
|
||||||
|
|
||||||
|
|
||||||
|
def load_definitions(keyword_definitions_file):
|
||||||
|
docs = {}
|
||||||
|
with open(keyword_definitions_file) as f:
|
||||||
|
docs = yaml.safe_load(f)
|
||||||
|
|
||||||
|
return docs
|
||||||
|
|
||||||
|
|
||||||
|
def extract_keywords(keyword_definitions):
|
||||||
|
pb_keywords = {}
|
||||||
|
for pb_class_name in PLAYBOOK_CLASS_NAMES:
|
||||||
|
if pb_class_name == 'Play':
|
||||||
|
module_name = 'ansible.playbook'
|
||||||
|
else:
|
||||||
|
module_name = 'ansible.playbook.{0}'.format(pb_class_name.lower())
|
||||||
|
module = importlib.import_module(module_name)
|
||||||
|
playbook_class = getattr(module, pb_class_name, None)
|
||||||
|
if playbook_class is None:
|
||||||
|
raise ImportError("We weren't able to import the module {0}".format(module_name))
|
||||||
|
|
||||||
|
# Maintain order of the actual class names for our output
|
||||||
|
# Build up a mapping of playbook classes to the attributes that they hold
|
||||||
|
pb_keywords[pb_class_name] = {k: v for (k, v) in playbook_class._valid_attrs.items()
|
||||||
|
# Filter private attributes as they're not usable in playbooks
|
||||||
|
if not v.private}
|
||||||
|
|
||||||
|
# pick up definitions if they exist
|
||||||
|
for keyword in tuple(pb_keywords[pb_class_name]):
|
||||||
|
if keyword in keyword_definitions:
|
||||||
|
pb_keywords[pb_class_name][keyword] = keyword_definitions[keyword]
|
||||||
|
else:
|
||||||
|
# check if there is an alias, otherwise undocumented
|
||||||
|
alias = getattr(getattr(playbook_class, '_%s' % keyword), 'alias', None)
|
||||||
|
if alias and alias in keyword_definitions:
|
||||||
|
pb_keywords[pb_class_name][alias] = keyword_definitions[alias]
|
||||||
|
del pb_keywords[pb_class_name][keyword]
|
||||||
|
else:
|
||||||
|
pb_keywords[pb_class_name][keyword] = ' UNDOCUMENTED!! '
|
||||||
|
|
||||||
|
# loop is really with_ for users
|
||||||
|
if pb_class_name == 'Task':
|
||||||
|
pb_keywords[pb_class_name]['with_<lookup_plugin>'] = (
|
||||||
|
'The same as ``loop`` but magically adds the output of any lookup plugin to'
|
||||||
|
' generate the item list.')
|
||||||
|
|
||||||
|
# local_action is implicit with action
|
||||||
|
if 'action' in pb_keywords[pb_class_name]:
|
||||||
|
pb_keywords[pb_class_name]['local_action'] = ('Same as action but also implies'
|
||||||
|
' ``delegate_to: localhost``')
|
||||||
|
|
||||||
|
return pb_keywords
|
||||||
|
|
||||||
|
|
||||||
|
def generate_page(pb_keywords, template_dir):
|
||||||
|
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
|
||||||
|
template = env.get_template(TEMPLATE_FILE)
|
||||||
|
tempvars = {'pb_keywords': pb_keywords, 'playbook_class_names': PLAYBOOK_CLASS_NAMES}
|
||||||
|
|
||||||
|
keyword_page = template.render(tempvars)
|
||||||
|
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
|
||||||
|
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
|
||||||
|
keyword_page = re.sub(' +\n', '\n', keyword_page)
|
||||||
|
|
||||||
|
return keyword_page
|
||||||
|
|
||||||
|
|
||||||
|
class DocumentKeywords(Command):
|
||||||
|
name = 'document-keywords'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def init_parser(cls, add_parser):
|
||||||
|
parser = add_parser(cls.name, description='Generate playbook keyword documentation from'
|
||||||
|
' code and descriptions')
|
||||||
|
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
|
||||||
|
default=DEFAULT_TEMPLATE_DIR,
|
||||||
|
help="directory containing Jinja2 templates")
|
||||||
|
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
|
||||||
|
default='/tmp/', help="Output directory for rst files")
|
||||||
|
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
|
||||||
|
help="Source for attribute docs")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def main(args):
|
||||||
|
if not args.docs:
|
||||||
|
print('Definitions for keywords must be specified via `--docs-source FILENAME`')
|
||||||
|
return 1
|
||||||
|
|
||||||
|
keyword_definitions = load_definitions(args.docs)
|
||||||
|
pb_keywords = extract_keywords(keyword_definitions)
|
||||||
|
|
||||||
|
keyword_page = generate_page(pb_keywords, args.template_dir)
|
||||||
|
outputname = os.path.join(args.output_dir, TEMPLATE_FILE.replace('.j2', ''))
|
||||||
|
update_file_if_different(outputname, to_bytes(keyword_page))
|
||||||
|
|
||||||
|
return 0
|
153
docs/bin/generate_man.py → hacking/build_library/build_ansible/command_plugins/generate_man.py
Executable file → Normal file
153
docs/bin/generate_man.py → hacking/build_library/build_ansible/command_plugins/generate_man.py
Executable file → Normal file
|
@ -1,25 +1,27 @@
|
||||||
#!/usr/bin/env python
|
# coding: utf-8
|
||||||
|
# Copyright: (c) 2019, Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os.path
|
||||||
|
import pathlib
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from jinja2 import Environment, FileSystemLoader
|
from jinja2 import Environment, FileSystemLoader
|
||||||
|
|
||||||
from ansible.module_utils._text import to_bytes
|
from ansible.module_utils._text import to_bytes
|
||||||
from ansible.utils._build_helpers import update_file_if_different
|
|
||||||
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
|
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||||
|
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
|
|
||||||
def generate_parser():
|
DEFAULT_TEMPLATE_FILE = pathlib.Path(__file__).parents[4] / 'docs/templates/man.j2'
|
||||||
p = argparse.ArgumentParser(
|
|
||||||
description='Generate cli documentation from cli docstrings',
|
|
||||||
)
|
|
||||||
|
|
||||||
p.add_argument("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
|
|
||||||
p.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
|
|
||||||
p.add_argument("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
|
|
||||||
p.add_argument('args', help='CLI module(s)', metavar='module', nargs='*')
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
# from https://www.python.org/dev/peps/pep-0257/
|
# from https://www.python.org/dev/peps/pep-0257/
|
||||||
|
@ -213,78 +215,89 @@ def opts_docs(cli_class_name, cli_module_name):
|
||||||
return docs
|
return docs
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
class GenerateMan(Command):
|
||||||
|
name = 'generate-man'
|
||||||
|
|
||||||
parser = generate_parser()
|
@classmethod
|
||||||
|
def init_parser(cls, add_parser):
|
||||||
|
parser = add_parser(name=cls.name,
|
||||||
|
description='Generate cli documentation from cli docstrings')
|
||||||
|
|
||||||
options = parser.parse_args()
|
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
|
||||||
|
default=DEFAULT_TEMPLATE_FILE, help="path to jinja2 template")
|
||||||
|
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
|
||||||
|
default='/tmp/', help="Output directory for rst files")
|
||||||
|
parser.add_argument("-f", "--output-format", action="store", dest="output_format",
|
||||||
|
default='man',
|
||||||
|
help="Output format for docs (the default 'man' or 'rst')")
|
||||||
|
parser.add_argument('cli_modules', help='CLI module name(s)', metavar='MODULE_NAME', nargs='*')
|
||||||
|
|
||||||
template_file = options.template_file
|
@staticmethod
|
||||||
template_path = os.path.expanduser(template_file)
|
def main(args):
|
||||||
template_dir = os.path.abspath(os.path.dirname(template_path))
|
template_file = args.template_file
|
||||||
template_basename = os.path.basename(template_file)
|
template_path = os.path.expanduser(template_file)
|
||||||
|
template_dir = os.path.abspath(os.path.dirname(template_path))
|
||||||
|
template_basename = os.path.basename(template_file)
|
||||||
|
|
||||||
output_dir = os.path.abspath(options.output_dir)
|
output_dir = os.path.abspath(args.output_dir)
|
||||||
output_format = options.output_format
|
output_format = args.output_format
|
||||||
|
|
||||||
cli_modules = options.args
|
cli_modules = args.cli_modules
|
||||||
|
|
||||||
# various cli parsing things checks sys.argv if the 'args' that are passed in are []
|
# various cli parsing things checks sys.argv if the 'args' that are passed in are []
|
||||||
# so just remove any args so the cli modules dont try to parse them resulting in warnings
|
# so just remove any args so the cli modules dont try to parse them resulting in warnings
|
||||||
sys.argv = [sys.argv[0]]
|
sys.argv = [sys.argv[0]]
|
||||||
# need to be in right dir
|
|
||||||
os.chdir(os.path.dirname(__file__))
|
|
||||||
|
|
||||||
allvars = {}
|
allvars = {}
|
||||||
output = {}
|
output = {}
|
||||||
cli_list = []
|
cli_list = []
|
||||||
cli_bin_name_list = []
|
cli_bin_name_list = []
|
||||||
|
|
||||||
# for binary in os.listdir('../../lib/ansible/cli'):
|
# for binary in os.listdir('../../lib/ansible/cli'):
|
||||||
for cli_module_name in cli_modules:
|
for cli_module_name in cli_modules:
|
||||||
binary = os.path.basename(os.path.expanduser(cli_module_name))
|
binary = os.path.basename(os.path.expanduser(cli_module_name))
|
||||||
|
|
||||||
if not binary.endswith('.py'):
|
if not binary.endswith('.py'):
|
||||||
continue
|
continue
|
||||||
elif binary == '__init__.py':
|
elif binary == '__init__.py':
|
||||||
continue
|
continue
|
||||||
|
|
||||||
cli_name = os.path.splitext(binary)[0]
|
cli_name = os.path.splitext(binary)[0]
|
||||||
|
|
||||||
if cli_name == 'adhoc':
|
if cli_name == 'adhoc':
|
||||||
cli_class_name = 'AdHocCLI'
|
cli_class_name = 'AdHocCLI'
|
||||||
# myclass = 'AdHocCLI'
|
# myclass = 'AdHocCLI'
|
||||||
output[cli_name] = 'ansible.1.rst.in'
|
output[cli_name] = 'ansible.1.rst.in'
|
||||||
cli_bin_name = 'ansible'
|
cli_bin_name = 'ansible'
|
||||||
else:
|
else:
|
||||||
# myclass = "%sCLI" % libname.capitalize()
|
# myclass = "%sCLI" % libname.capitalize()
|
||||||
cli_class_name = "%sCLI" % cli_name.capitalize()
|
cli_class_name = "%sCLI" % cli_name.capitalize()
|
||||||
output[cli_name] = 'ansible-%s.1.rst.in' % cli_name
|
output[cli_name] = 'ansible-%s.1.rst.in' % cli_name
|
||||||
cli_bin_name = 'ansible-%s' % cli_name
|
cli_bin_name = 'ansible-%s' % cli_name
|
||||||
|
|
||||||
# FIXME:
|
# FIXME:
|
||||||
allvars[cli_name] = opts_docs(cli_class_name, cli_name)
|
allvars[cli_name] = opts_docs(cli_class_name, cli_name)
|
||||||
cli_bin_name_list.append(cli_bin_name)
|
cli_bin_name_list.append(cli_bin_name)
|
||||||
|
|
||||||
cli_list = allvars.keys()
|
cli_list = allvars.keys()
|
||||||
|
|
||||||
doc_name_formats = {'man': '%s.1.rst.in',
|
doc_name_formats = {'man': '%s.1.rst.in',
|
||||||
'rst': '%s.rst'}
|
'rst': '%s.rst'}
|
||||||
|
|
||||||
for cli_name in cli_list:
|
for cli_name in cli_list:
|
||||||
|
|
||||||
# template it!
|
# template it!
|
||||||
env = Environment(loader=FileSystemLoader(template_dir))
|
env = Environment(loader=FileSystemLoader(template_dir))
|
||||||
template = env.get_template(template_basename)
|
template = env.get_template(template_basename)
|
||||||
|
|
||||||
# add rest to vars
|
# add rest to vars
|
||||||
tvars = allvars[cli_name]
|
tvars = allvars[cli_name]
|
||||||
tvars['cli_list'] = cli_list
|
tvars['cli_list'] = cli_list
|
||||||
tvars['cli_bin_name_list'] = cli_bin_name_list
|
tvars['cli_bin_name_list'] = cli_bin_name_list
|
||||||
tvars['cli'] = cli_name
|
tvars['cli'] = cli_name
|
||||||
if '-i' in tvars['options']:
|
if '-i' in tvars['options']:
|
||||||
print('uses inventory')
|
print('uses inventory')
|
||||||
|
|
||||||
manpage = template.render(tvars)
|
manpage = template.render(tvars)
|
||||||
filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
|
filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
|
||||||
update_file_if_different(filename, to_bytes(manpage))
|
update_file_if_different(filename, to_bytes(manpage))
|
177
docs/bin/plugin_formatter.py → hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
Executable file → Normal file
177
docs/bin/plugin_formatter.py → hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
Executable file → Normal file
|
@ -1,4 +1,3 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
|
# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
|
||||||
# Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
|
# Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
|
||||||
# Copyright: (c) 2017, Ansible Project
|
# Copyright: (c) 2017, Ansible Project
|
||||||
|
@ -45,7 +44,10 @@ from ansible.module_utils.six import iteritems, string_types
|
||||||
from ansible.plugins.loader import fragment_loader
|
from ansible.plugins.loader import fragment_loader
|
||||||
from ansible.utils import plugin_docs
|
from ansible.utils import plugin_docs
|
||||||
from ansible.utils.display import Display
|
from ansible.utils.display import Display
|
||||||
from ansible.utils._build_helpers import update_file_if_different
|
|
||||||
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
|
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
|
||||||
|
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
|
|
||||||
#####################################################################################
|
#####################################################################################
|
||||||
|
@ -363,29 +365,6 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
|
||||||
return module_info, categories
|
return module_info, categories
|
||||||
|
|
||||||
|
|
||||||
def generate_parser():
|
|
||||||
''' generate an optparse parser '''
|
|
||||||
|
|
||||||
p = optparse.OptionParser(
|
|
||||||
version='%prog 1.0',
|
|
||||||
usage='usage: %prog [options] arg1 arg2',
|
|
||||||
description='Generate module documentation from metadata',
|
|
||||||
)
|
|
||||||
|
|
||||||
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
|
|
||||||
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
|
|
||||||
p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)")
|
|
||||||
p.add_option("-T", "--template-dir", action="append", dest="template_dir", help="directory containing Jinja2 templates")
|
|
||||||
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
|
|
||||||
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
|
|
||||||
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
|
|
||||||
p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None,
|
|
||||||
help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.")
|
|
||||||
p.add_option('-V', action='version', help='Show version number and exit')
|
|
||||||
p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)")
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def jinja2_environment(template_dir, typ, plugin_type):
|
def jinja2_environment(template_dir, typ, plugin_type):
|
||||||
|
|
||||||
env = Environment(loader=FileSystemLoader(template_dir),
|
env = Environment(loader=FileSystemLoader(template_dir),
|
||||||
|
@ -734,82 +713,106 @@ def validate_options(options):
|
||||||
sys.exit("--template-dir must be specified")
|
sys.exit("--template-dir must be specified")
|
||||||
|
|
||||||
|
|
||||||
def main():
|
class DocumentPlugins(Command):
|
||||||
|
name = 'document-plugins'
|
||||||
|
|
||||||
# INIT
|
@classmethod
|
||||||
p = generate_parser()
|
def init_parser(cls, add_parser):
|
||||||
(options, args) = p.parse_args()
|
parser = add_parser(cls.name, description='Generate module documentation from metadata')
|
||||||
if not options.template_dir:
|
|
||||||
options.template_dir = ["hacking/templates"]
|
|
||||||
validate_options(options)
|
|
||||||
display.verbosity = options.verbosity
|
|
||||||
plugin_type = options.plugin_type
|
|
||||||
|
|
||||||
display.display("Evaluating %s files..." % plugin_type)
|
parser.add_argument("-A", "--ansible-version", action="store", dest="ansible_version",
|
||||||
|
default="unknown", help="Ansible version number")
|
||||||
|
parser.add_argument("-M", "--module-dir", action="store", dest="module_dir",
|
||||||
|
default=MODULEDIR, help="Ansible library path")
|
||||||
|
parser.add_argument("-P", "--plugin-type", action="store", dest="plugin_type",
|
||||||
|
default='module', help="The type of plugin (module, lookup, etc)")
|
||||||
|
parser.add_argument("-T", "--template-dir", action="append", dest="template_dir",
|
||||||
|
help="directory containing Jinja2 templates")
|
||||||
|
parser.add_argument("-t", "--type", action='store', dest='type', choices=['rst'],
|
||||||
|
default='rst', help="Document type")
|
||||||
|
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default=None,
|
||||||
|
help="Output directory for module files")
|
||||||
|
parser.add_argument("-I", "--includes-file", action="store", dest="includes_file",
|
||||||
|
default=None, help="Create a file containing list of processed modules")
|
||||||
|
parser.add_argument("-l", "--limit-to-modules", '--limit-to', action="store",
|
||||||
|
dest="limit_to", default=None, help="Limit building module documentation"
|
||||||
|
" to comma-separated list of plugins. Specify non-existing plugin name"
|
||||||
|
" for no plugins.")
|
||||||
|
parser.add_argument('-V', action='version', help='Show version number and exit')
|
||||||
|
parser.add_argument('-v', '--verbose', dest='verbosity', default=0, action="count",
|
||||||
|
help="verbose mode (increase number of 'v's for more)")
|
||||||
|
|
||||||
# prep templating
|
@staticmethod
|
||||||
templates = jinja2_environment(options.template_dir, options.type, plugin_type)
|
def main(args):
|
||||||
|
if not args.template_dir:
|
||||||
|
args.template_dir = ["hacking/templates"]
|
||||||
|
validate_options(args)
|
||||||
|
display.verbosity = args.verbosity
|
||||||
|
plugin_type = args.plugin_type
|
||||||
|
|
||||||
# set file/directory structure
|
display.display("Evaluating %s files..." % plugin_type)
|
||||||
if plugin_type == 'module':
|
|
||||||
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
|
|
||||||
outputname = '%s_' + '%s.rst' % plugin_type
|
|
||||||
output_dir = options.output_dir
|
|
||||||
else:
|
|
||||||
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
|
|
||||||
outputname = '%s.rst'
|
|
||||||
output_dir = '%s/plugins/%s' % (options.output_dir, plugin_type)
|
|
||||||
|
|
||||||
display.vv('output name: %s' % outputname)
|
# prep templating
|
||||||
display.vv('output dir: %s' % output_dir)
|
templates = jinja2_environment(args.template_dir, args.type, plugin_type)
|
||||||
|
|
||||||
# Convert passed-in limit_to to None or list of modules.
|
# set file/directory structure
|
||||||
if options.limit_to is not None:
|
if plugin_type == 'module':
|
||||||
options.limit_to = [s.lower() for s in options.limit_to.split(",")]
|
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
|
||||||
|
outputname = '%s_' + '%s.rst' % plugin_type
|
||||||
|
output_dir = args.output_dir
|
||||||
|
else:
|
||||||
|
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
|
||||||
|
outputname = '%s.rst'
|
||||||
|
output_dir = '%s/plugins/%s' % (args.output_dir, plugin_type)
|
||||||
|
|
||||||
plugin_info, categories = get_plugin_info(options.module_dir, limit_to=options.limit_to, verbose=(options.verbosity > 0))
|
display.vv('output name: %s' % outputname)
|
||||||
|
display.vv('output dir: %s' % output_dir)
|
||||||
|
|
||||||
categories['all'] = {'_modules': plugin_info.keys()}
|
# Convert passed-in limit_to to None or list of modules.
|
||||||
|
if args.limit_to is not None:
|
||||||
|
args.limit_to = [s.lower() for s in args.limit_to.split(",")]
|
||||||
|
|
||||||
if display.verbosity >= 3:
|
plugin_info, categories = get_plugin_info(args.module_dir, limit_to=args.limit_to, verbose=(args.verbosity > 0))
|
||||||
display.vvv(pp.pformat(categories))
|
|
||||||
if display.verbosity >= 5:
|
|
||||||
display.vvvvv(pp.pformat(plugin_info))
|
|
||||||
|
|
||||||
# Transform the data
|
categories['all'] = {'_modules': plugin_info.keys()}
|
||||||
if options.type == 'rst':
|
|
||||||
display.v('Generating rst')
|
|
||||||
for key, record in plugin_info.items():
|
|
||||||
display.vv(key)
|
|
||||||
if display.verbosity >= 5:
|
|
||||||
display.vvvvv(pp.pformat(('record', record)))
|
|
||||||
if record.get('doc', None):
|
|
||||||
short_desc = record['doc']['short_description'].rstrip('.')
|
|
||||||
if short_desc is None:
|
|
||||||
display.warning('short_description for %s is None' % key)
|
|
||||||
short_desc = ''
|
|
||||||
record['doc']['short_description'] = rst_ify(short_desc)
|
|
||||||
|
|
||||||
if plugin_type == 'module':
|
if display.verbosity >= 3:
|
||||||
display.v('Generating Categories')
|
display.vvv(pp.pformat(categories))
|
||||||
# Write module master category list
|
if display.verbosity >= 5:
|
||||||
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
|
display.vvvvv(pp.pformat(plugin_info))
|
||||||
category_index_name = '%ss_by_category.rst' % plugin_type
|
|
||||||
write_data(category_list_text, output_dir, category_index_name)
|
|
||||||
|
|
||||||
# Render all the individual plugin pages
|
# Transform the data
|
||||||
display.v('Generating plugin pages')
|
if args.type == 'rst':
|
||||||
process_plugins(plugin_info, templates, outputname, output_dir, options.ansible_version, plugin_type)
|
display.v('Generating rst')
|
||||||
|
for key, record in plugin_info.items():
|
||||||
|
display.vv(key)
|
||||||
|
if display.verbosity >= 5:
|
||||||
|
display.vvvvv(pp.pformat(('record', record)))
|
||||||
|
if record.get('doc', None):
|
||||||
|
short_desc = record['doc']['short_description'].rstrip('.')
|
||||||
|
if short_desc is None:
|
||||||
|
display.warning('short_description for %s is None' % key)
|
||||||
|
short_desc = ''
|
||||||
|
record['doc']['short_description'] = rst_ify(short_desc)
|
||||||
|
|
||||||
# Render all the categories for modules
|
if plugin_type == 'module':
|
||||||
if plugin_type == 'module':
|
display.v('Generating Categories')
|
||||||
display.v('Generating Category lists')
|
# Write module master category list
|
||||||
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
|
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
|
||||||
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
|
category_index_name = '%ss_by_category.rst' % plugin_type
|
||||||
|
write_data(category_list_text, output_dir, category_index_name)
|
||||||
|
|
||||||
|
# Render all the individual plugin pages
|
||||||
|
display.v('Generating plugin pages')
|
||||||
|
process_plugins(plugin_info, templates, outputname, output_dir, args.ansible_version, plugin_type)
|
||||||
|
|
||||||
# Render all the categories for modules
|
# Render all the categories for modules
|
||||||
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
|
if plugin_type == 'module':
|
||||||
|
display.v('Generating Category lists')
|
||||||
|
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
|
||||||
|
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
|
||||||
|
|
||||||
|
# Render all the categories for modules
|
||||||
|
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
return 0
|
||||||
main()
|
|
|
@ -121,7 +121,7 @@ def generate_porting_guide(version):
|
||||||
|
|
||||||
|
|
||||||
def write_guide(version, guide_content):
|
def write_guide(version, guide_content):
|
||||||
filename = f'porting_guide_{version}.rst'
|
filename = 'porting_guide_{0}.rst'.format(version)
|
||||||
with open(filename, 'w') as out_file:
|
with open(filename, 'w') as out_file:
|
||||||
out_file.write(guide_content)
|
out_file.write(guide_content)
|
||||||
|
|
||||||
|
|
|
@ -7,148 +7,13 @@ from __future__ import (absolute_import, division, print_function)
|
||||||
__metaclass__ = type
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import datetime
|
|
||||||
import hashlib
|
|
||||||
import os.path
|
|
||||||
import sys
|
import sys
|
||||||
from collections import UserString
|
from collections import UserString
|
||||||
from distutils.version import LooseVersion
|
from distutils.version import LooseVersion
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from jinja2 import Environment, DictLoader
|
|
||||||
|
|
||||||
# Pylint doesn't understand Python3 namespace modules.
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
from ..commands import Command # pylint: disable=relative-beyond-top-level
|
||||||
|
from .. import errors # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
# pylint: disable=
|
|
||||||
VERSION_FRAGMENT = """
|
|
||||||
{%- if versions | length > 1 %}
|
|
||||||
{% for version in versions %}
|
|
||||||
{% if loop.last %}and {{ version }}{% else %}
|
|
||||||
{% if versions | length == 2 %}{{ version }} {% else %}{{ version }}, {% endif -%}
|
|
||||||
{% endif -%}
|
|
||||||
{% endfor -%}
|
|
||||||
{%- else %}{{ versions[0] }}{% endif -%}
|
|
||||||
"""
|
|
||||||
|
|
||||||
LONG_TEMPLATE = """
|
|
||||||
{% set plural = False if versions | length == 1 else True %}
|
|
||||||
{% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
|
|
||||||
|
|
||||||
To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
|
|
||||||
Subject: New Ansible release{% if plural %}s{% endif %} {{ version_str }}
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
Hi all- we're happy to announce that the general release of Ansible {{ version_str }}{% if plural %} are{%- else %} is{%- endif %} now available!
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
How do you get it?
|
|
||||||
------------------
|
|
||||||
|
|
||||||
{% for version in versions %}
|
|
||||||
$ pip install ansible=={{ version }} --user
|
|
||||||
{% if not loop.last %}
|
|
||||||
or
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
The tar.gz of the release{% if plural %}s{% endif %} can be found here:
|
|
||||||
|
|
||||||
{% for version in versions %}
|
|
||||||
* {{ version }}
|
|
||||||
https://releases.ansible.com/ansible/ansible-{{ version }}.tar.gz
|
|
||||||
SHA256: {{ hashes[version] }}
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
|
|
||||||
What's new in {{ version_str }}
|
|
||||||
{{ '-' * (14 + version_str | length) }}
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
{% for version in versions %}
|
|
||||||
* {{ version }}
|
|
||||||
https://github.com/ansible/ansible/blob/stable-{{ version.split('.')[:2] | join('.') }}/changelogs/CHANGELOG-v{{ version.split('.')[:2] | join('.') }}.rst
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
|
|
||||||
What's the schedule for future maintenance releases?
|
|
||||||
----------------------------------------------------
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
Future maintenance releases will occur approximately every 3 weeks. So expect the next one around {{ next_release.strftime('%Y-%m-%d') }}.
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Porting Help
|
|
||||||
------------
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
We've published a porting guide at
|
|
||||||
https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_{{ latest_ver.split('.')[:2] | join('.') }}.html to help migrate your content to {{ latest_ver.split('.')[:2] | join('.') }}.
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
If you discover any errors or if any of your working playbooks break when you upgrade to {{ latest_ver }}, please use the following link to report the regression:
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
https://github.com/ansible/ansible/issues/new/choose
|
|
||||||
|
|
||||||
{% filter wordwrap %}
|
|
||||||
In your issue, be sure to mention the Ansible version that works and the one that doesn't.
|
|
||||||
{% endfilter %}
|
|
||||||
|
|
||||||
|
|
||||||
Thanks!
|
|
||||||
|
|
||||||
-{{ name }}
|
|
||||||
|
|
||||||
""" # noqa for E501 (line length).
|
|
||||||
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
|
|
||||||
# proper wrapping to occur
|
|
||||||
|
|
||||||
SHORT_TEMPLATE = """
|
|
||||||
{% set plural = False if versions | length == 1 else True %}
|
|
||||||
@ansible
|
|
||||||
{{ version_str }}
|
|
||||||
{% if plural %}
|
|
||||||
have
|
|
||||||
{% else %}
|
|
||||||
has
|
|
||||||
{% endif %}
|
|
||||||
been released! Get
|
|
||||||
{% if plural %}
|
|
||||||
them
|
|
||||||
{% else %}
|
|
||||||
it
|
|
||||||
{% endif %}
|
|
||||||
on PyPI: pip install ansible=={{ (versions|sort(attribute='ver_obj'))[-1] }},
|
|
||||||
https://releases.ansible.com/ansible/, the Ansible PPA on Launchpad, or GitHub. Happy automating!
|
|
||||||
""" # noqa for E501 (line length).
|
|
||||||
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
|
|
||||||
# proper wrapping to occur
|
|
||||||
|
|
||||||
JINJA_ENV = Environment(
|
|
||||||
loader=DictLoader({'long': LONG_TEMPLATE,
|
|
||||||
'short': SHORT_TEMPLATE,
|
|
||||||
'version_string': VERSION_FRAGMENT,
|
|
||||||
}),
|
|
||||||
extensions=['jinja2.ext.i18n'],
|
|
||||||
trim_blocks=True,
|
|
||||||
lstrip_blocks=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class VersionStr(UserString):
|
class VersionStr(UserString):
|
||||||
|
@ -167,108 +32,6 @@ def transform_args(args):
|
||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
async def calculate_hash_from_tarball(session, version):
|
|
||||||
tar_url = f'https://releases.ansible.com/ansible/ansible-{version}.tar.gz'
|
|
||||||
tar_task = asyncio.create_task(session.get(tar_url))
|
|
||||||
tar_response = await tar_task
|
|
||||||
|
|
||||||
tar_hash = hashlib.sha256()
|
|
||||||
while True:
|
|
||||||
chunk = await tar_response.content.read(1024)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
tar_hash.update(chunk)
|
|
||||||
|
|
||||||
return tar_hash.hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
async def parse_hash_from_file(session, version):
|
|
||||||
filename = f'ansible-{version}.tar.gz'
|
|
||||||
hash_url = f'https://releases.ansible.com/ansible/{filename}.sha'
|
|
||||||
hash_task = asyncio.create_task(session.get(hash_url))
|
|
||||||
hash_response = await hash_task
|
|
||||||
|
|
||||||
hash_content = await hash_response.read()
|
|
||||||
precreated_hash, precreated_filename = hash_content.split(None, 1)
|
|
||||||
if filename != precreated_filename.strip().decode('utf-8'):
|
|
||||||
raise ValueError(f'Hash file contains hash for a different file: {precreated_filename}')
|
|
||||||
|
|
||||||
return precreated_hash.decode('utf-8')
|
|
||||||
|
|
||||||
|
|
||||||
async def get_hash(session, version):
|
|
||||||
calculated_hash = await calculate_hash_from_tarball(session, version)
|
|
||||||
precreated_hash = await parse_hash_from_file(session, version)
|
|
||||||
|
|
||||||
if calculated_hash != precreated_hash:
|
|
||||||
raise ValueError(f'Hash in file ansible-{version}.tar.gz.sha {precreated_hash} does not'
|
|
||||||
f' match hash of tarball {calculated_hash}')
|
|
||||||
|
|
||||||
return calculated_hash
|
|
||||||
|
|
||||||
|
|
||||||
async def get_hashes(versions):
|
|
||||||
hashes = {}
|
|
||||||
requestors = {}
|
|
||||||
async with aiohttp.ClientSession() as aio_session:
|
|
||||||
for version in versions:
|
|
||||||
requestors[version] = asyncio.create_task(get_hash(aio_session, version))
|
|
||||||
|
|
||||||
for version, request in requestors.items():
|
|
||||||
await request
|
|
||||||
hashes[version] = request.result()
|
|
||||||
|
|
||||||
return hashes
|
|
||||||
|
|
||||||
|
|
||||||
def next_release_date(weeks=3):
|
|
||||||
days_in_the_future = weeks * 7
|
|
||||||
today = datetime.datetime.now()
|
|
||||||
numeric_today = today.weekday()
|
|
||||||
|
|
||||||
# We release on Thursdays
|
|
||||||
if numeric_today == 3:
|
|
||||||
# 3 is Thursday
|
|
||||||
pass
|
|
||||||
elif numeric_today == 4:
|
|
||||||
# If this is Friday, we can adjust back to Thursday for the next release
|
|
||||||
today -= datetime.timedelta(days=1)
|
|
||||||
elif numeric_today < 3:
|
|
||||||
# Otherwise, slide forward to Thursday
|
|
||||||
today += datetime.timedelta(days=(3 - numeric_today))
|
|
||||||
else:
|
|
||||||
# slightly different formula if it's past Thursday this week. We need to go forward to
|
|
||||||
# Thursday of next week
|
|
||||||
today += datetime.timedelta(days=(10 - numeric_today))
|
|
||||||
|
|
||||||
next_release = today + datetime.timedelta(days=days_in_the_future)
|
|
||||||
return next_release
|
|
||||||
|
|
||||||
|
|
||||||
def generate_long_message(versions, name):
|
|
||||||
hashes = asyncio.run(get_hashes(versions))
|
|
||||||
|
|
||||||
version_template = JINJA_ENV.get_template('version_string')
|
|
||||||
version_str = version_template.render(versions=versions).strip()
|
|
||||||
|
|
||||||
next_release = next_release_date()
|
|
||||||
|
|
||||||
template = JINJA_ENV.get_template('long')
|
|
||||||
message = template.render(versions=versions, version_str=version_str,
|
|
||||||
name=name, hashes=hashes, next_release=next_release)
|
|
||||||
return message
|
|
||||||
|
|
||||||
|
|
||||||
def generate_short_message(versions):
|
|
||||||
version_template = JINJA_ENV.get_template('version_string')
|
|
||||||
version_str = version_template.render(versions=versions).strip()
|
|
||||||
|
|
||||||
template = JINJA_ENV.get_template('short')
|
|
||||||
message = template.render(versions=versions, version_str=version_str)
|
|
||||||
message = ' '.join(message.split()) + '\n'
|
|
||||||
return message
|
|
||||||
|
|
||||||
|
|
||||||
def write_message(filename, message):
|
def write_message(filename, message):
|
||||||
if filename != '-':
|
if filename != '-':
|
||||||
with open(filename, 'w') as out_file:
|
with open(filename, 'w') as out_file:
|
||||||
|
@ -294,12 +57,21 @@ class ReleaseAnnouncementCommand(Command):
|
||||||
parser.add_argument("--twitter-out", type=str, default="-",
|
parser.add_argument("--twitter-out", type=str, default="-",
|
||||||
help="Filename to place the twitter announcement into")
|
help="Filename to place the twitter announcement into")
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def main(args):
|
def main(cls, args):
|
||||||
|
if sys.version_info < (3, 6):
|
||||||
|
raise errors.DependencyError('The {0} subcommand needs Python-3.6+'
|
||||||
|
' to run'.format(cls.name))
|
||||||
|
|
||||||
|
# Import here because these functions are invalid on Python-3.5 and the command plugins and
|
||||||
|
# init_parser() method need to be compatible with Python-3.4+ for now.
|
||||||
|
# Pylint doesn't understand Python3 namespace modules.
|
||||||
|
from .. announce import create_short_message, create_long_message # pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
args = transform_args(args)
|
args = transform_args(args)
|
||||||
|
|
||||||
twitter_message = generate_short_message(args.versions)
|
twitter_message = create_short_message(args.versions)
|
||||||
email_message = generate_long_message(args.versions, args.name)
|
email_message = create_long_message(args.versions, args.name)
|
||||||
|
|
||||||
write_message(args.twitter_out, twitter_message)
|
write_message(args.twitter_out, twitter_message)
|
||||||
write_message(args.email_out, email_message)
|
write_message(args.email_out, email_message)
|
||||||
|
|
12
hacking/build_library/build_ansible/errors.py
Normal file
12
hacking/build_library/build_ansible/errors.py
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
# coding: utf-8
|
||||||
|
# Copyright: (c) 2019, Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
# Make coding more python3-ish
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
class DependencyError(Exception):
|
||||||
|
"""Used when a dependency is unmet"""
|
||||||
|
pass
|
|
@ -52,8 +52,9 @@ class Attribute:
|
||||||
:kwarg isa: The type of the attribute. Allowable values are a string
|
:kwarg isa: The type of the attribute. Allowable values are a string
|
||||||
representation of any yaml basic datatype, python class, or percent.
|
representation of any yaml basic datatype, python class, or percent.
|
||||||
(Enforced at post-validation time).
|
(Enforced at post-validation time).
|
||||||
:kwarg private: Hides the attribute from being documented.
|
:kwarg private: Not used at runtime. The docs playbook keyword dumper uses it to determine
|
||||||
TODO: it should also should prevent it from being user settable
|
that a keyword should not be documented. mpdehaan had plans to remove attributes marked
|
||||||
|
private from the ds so they would not have been available at all.
|
||||||
:kwarg default: Default value if unspecified in the YAML document.
|
:kwarg default: Default value if unspecified in the YAML document.
|
||||||
:kwarg required: Whether or not the YAML document must contain this field.
|
:kwarg required: Whether or not the YAML document must contain this field.
|
||||||
If the attribute is None when post-validated, an error will be raised.
|
If the attribute is None when post-validated, an error will be raised.
|
||||||
|
|
|
@ -1,3 +1,11 @@
|
||||||
# The following are only run by release engineers who can be asked to have newer Python3 on their systems
|
# The following are only run by release engineers who can be asked to have newer Python3 on their systems
|
||||||
hacking/build_library/build_ansible/command_plugins/porting_guide.py
|
hacking/build_library/build_ansible/command_plugins/porting_guide.py
|
||||||
hacking/build_library/build_ansible/command_plugins/release_announcement.py
|
hacking/build_library/build_ansible/command_plugins/release_announcement.py
|
||||||
|
|
||||||
|
# The following are used to build docs. Since we explicitly say that the controller won't run on
|
||||||
|
# Python-2.6 (docs are built controller-side) and EPEL-6, the only LTS platform with Python-2.6,
|
||||||
|
# doesn't have a new enough sphinx to build docs, do not test these under Python-2.6
|
||||||
|
hacking/build_library/build_ansible/command_plugins/dump_config.py
|
||||||
|
hacking/build_library/build_ansible/command_plugins/dump_keywords.py
|
||||||
|
hacking/build_library/build_ansible/command_plugins/generate_man.py
|
||||||
|
hacking/build_library/build_ansible/command_plugins/plugin_formatter.py
|
||||||
|
|
Loading…
Reference in a new issue