cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move common build code from _build_helpers (#55986)

Browse source 
We have some common code used by several docs scripts.  Migrate that
into the build-only shared code repository.

* Move lib/ansible/utils/_build_helpers.py to the directory for common
  build code
* Migrate docs/bin/dump_config.py to a build-ansible subcommand
* Migrate dump_keywords to the build-ansible framework
  * Make the script more maintainable by using functions and good
    variable names
  * Port to Python3 idioms
  * Fix bug so that private attributes will be undocumented
* Move generate_man to a build-ansible subcommand
* Port plugin_formatter to a build-ansible subcommand
* Rework command_plugins so that docs scripts can target Python-3.4+ and
  releng-only subcommands can use more recent versions of Python.
  The architecture is now that command_plugins/* need to be importable
  on Python-3.4.  The init_parsers() method needs to run on Python-3.4.
  But the main() method can utilize features of more recent Python as
  long as it fits within those parameters.
* Update docs build requirements

Port the plugin_formatter to build-ansible framework
This commit is contained in:
Toshio Kuratomi 2019-07-16 12:19:01 -07:00 committed by GitHub
parent 65e0f37fc0
commit 019d078a5a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 457 additions and 586 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -33,7 +33,7 @@ ASCII2MAN = @echo "ERROR: rst2man from docutils command is not installed but is
endif
PYTHON=python
GENERATE_CLI = $(PYTHON) docs/bin/generate_man.py
GENERATE_CLI = hacking/build-ansible.py generate-man
SITELIB = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")

74
docs/bin/dump_config.py
View file

@ -1,74 +0,0 @@
#!/usr/bin/env python
import optparse
import os
import sys
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
from ansible.utils._build_helpers import update_file_if_different
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
def generate_parser():
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options]',
description='Generate module documentation from metadata',
)
p.add_option("-t", "--template-file", action="store", dest="template_file", default=DEFAULT_TEMPLATE_FILE, help="directory containing Jinja2 templates")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
(options, args) = p.parse_args()
return p
def fix_description(config_options):
'''some descriptions are strings, some are lists. workaround it...'''
for config_key in config_options:
description = config_options[config_key].get('description', [])
if isinstance(description, list):
desc_list = description
else:
desc_list = [description]
config_options[config_key]['description'] = desc_list
return config_options
def main(args):
parser = generate_parser()
(options, args) = parser.parse_args()
output_dir = os.path.abspath(options.output_dir)
template_file_full_path = os.path.abspath(options.template_file)
template_file = os.path.basename(template_file_full_path)
template_dir = os.path.dirname(os.path.abspath(template_file_full_path))
if options.docs:
with open(options.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
config_options = docs
config_options = fix_description(config_options)
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(template_file)
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
temp_vars = {'config_options': config_options}
data = to_bytes(template.render(temp_vars))
update_file_if_different(output_name, data)
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[:]))

84
docs/bin/dump_keywords.py
View file

@ -1,84 +0,0 @@
#!/usr/bin/env python
import optparse
import re
from distutils.version import LooseVersion
import jinja2
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
from ansible.playbook import Play
from ansible.playbook.block import Block
from ansible.playbook.role import Role
from ansible.playbook.task import Task
from ansible.utils._build_helpers import update_file_if_different
template_file = 'playbooks_keywords.rst.j2'
oblist = {}
clist = []
class_list = [Play, Role, Block, Task]
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options]',
description='Generate playbook keyword documentation from code and descriptions',
)
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="../templates", help="directory containing Jinja2 templates")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_option("-d", "--docs-source", action="store", dest="docs", default=None, help="Source for attribute docs")
(options, args) = p.parse_args()
for aclass in class_list:
aobj = aclass()
name = type(aobj).__name__
if options.docs:
with open(options.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
# build ordered list to loop over and dict with attributes
clist.append(name)
oblist[name] = dict((x, aobj.__dict__['_attributes'][x]) for x in aobj.__dict__['_attributes'] if 'private' not in x or not x.private)
# pick up docs if they exist
for a in oblist[name]:
if a in docs:
oblist[name][a] = docs[a]
else:
# check if there is an alias, otherwise undocumented
alias = getattr(getattr(aobj, '_%s' % a), 'alias', None)
if alias and alias in docs:
oblist[name][alias] = docs[alias]
del oblist[name][a]
else:
oblist[name][a] = ' UNDOCUMENTED!! '
# loop is really with_ for users
if name == 'Task':
oblist[name]['with_<lookup_plugin>'] = 'The same as ``loop`` but magically adds the output of any lookup plugin to generate the item list.'
# local_action is implicit with action
if 'action' in oblist[name]:
oblist[name]['local_action'] = 'Same as action but also implies ``delegate_to: localhost``'
# remove unusable (used to be private?)
for nouse in ('loop_args', 'loop_with'):
if nouse in oblist[name]:
del oblist[name][nouse]
env = Environment(loader=FileSystemLoader(options.template_dir), trim_blocks=True,)
template = env.get_template(template_file)
outputname = options.output_dir + template_file.replace('.j2', '')
tempvars = {'oblist': oblist, 'clist': clist}
keyword_page = template.render(tempvars)
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
keyword_page = re.sub(' +\n', '\n', keyword_page)
update_file_if_different(outputname, to_bytes(keyword_page))

24
docs/docsite/Makefile
View file

@ -1,10 +1,10 @@
OS := $(shell uname -s)
SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
FORMATTER=../bin/plugin_formatter.py
PLUGIN_FORMATTER=../../hacking/build-ansible.py document-plugins
TESTING_FORMATTER=../bin/testing_formatter.sh
DUMPER=../bin/dump_keywords.py
CONFIG_DUMPER=../bin/dump_config.py
GENERATE_CLI=../bin/generate_man.py
KEYWORD_DUMPER=../../hacking/build-ansible.py document-keywords
CONFIG_DUMPER=../../hacking/build-ansible.py document-config
GENERATE_CLI=../../hacking/build-ansible.py generate-man
ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
else
@ -79,24 +79,24 @@ clean:
.PHONY: docs clean
# TODO: make generate_man output dir cli option
cli: $(GENERATE_CLI)
cli:
mkdir -p rst/cli
PYTHONPATH=../../lib $(GENERATE_CLI) --template-file=../templates/cli_rst.j2 --output-dir=rst/cli/ --output-format rst ../../lib/ansible/cli/*.py
keywords: $(FORMATTER) ../templates/playbooks_keywords.rst.j2
PYTHONPATH=../../lib $(DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
keywords: ../templates/playbooks_keywords.rst.j2
PYTHONPATH=../../lib $(KEYWORD_DUMPER) --template-dir=../templates --output-dir=rst/reference_appendices/ -d ./keyword_desc.yml
config:
config: ../templates/config.rst.j2
PYTHONPATH=../../lib $(CONFIG_DUMPER) --template-file=../templates/config.rst.j2 --output-dir=rst/reference_appendices/ -d ../../lib/ansible/config/base.yml
modules: $(FORMATTER) ../templates/plugin.rst.j2
PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
modules: ../templates/plugin.rst.j2
PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/modules/ $(MODULE_ARGS)
plugins: $(FORMATTER) ../templates/plugin.rst.j2
plugins: ../templates/plugin.rst.j2
@echo "looping over doc plugins"
for plugin in $(DOC_PLUGINS); \
do \
PYTHONPATH=../../lib $(FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
PYTHONPATH=../../lib $(PLUGIN_FORMATTER) -t rst --plugin-type $$plugin --template-dir=../templates --module-dir=../../lib/ansible/plugins/$$plugin -o rst $(PLUGIN_ARGS); \
done
testing:

1
docs/docsite/requirements.txt
View file

@ -5,3 +5,4 @@ rstcheck
sphinx
sphinx-notfound-page
Pygments >= 2.4.0
straight.plugin # Needed for hacking/build-ansible.py which is the backend build script

4
docs/docsite/rst/community/documentation_contributions.rst
View file

@ -60,7 +60,8 @@ If you make multiple changes to the documentation, or add more than a line to it
#. Test your changes for rST errors.
#. Build the page, and preferably the entire documentation site, locally.
To work with documentation on your local machine, you need the following packages installed:
To work with documentation on your local machine, you need to have python-3.5 or greater and the
following packages installed:
- gcc
- jinja2
@ -72,6 +73,7 @@ To work with documentation on your local machine, you need the following package
- six
- sphinx
- sphinx-notfound-page
- straight.plugin
.. note::

6
docs/templates/playbooks_keywords.rst.j2 vendored
View file

@ -19,15 +19,15 @@ These are the keywords available on common playbook objects. Keywords are one of
:local:
:depth: 1
{% for name in clist %}
{% for name in playbook_class_names %}
{{ name }}
{{ '-' * name|length }}
.. glossary::
{% for attribute in oblist[name]|sort %}
{% for attribute in pb_keywords[name]|sort %}
{{ attribute }}
{{ oblist[name][attribute] |indent(8) }}
{{ pb_keywords[name][attribute] |indent(8) }}
{% endfor %}
{% endfor %}

30
hacking/build-ansible.py
View file

@ -21,18 +21,17 @@ except ImportError:
argcomplete = None
def set_sys_path(this_script=__file__):
"""Add path to the common librarydirectory to :attr:`sys.path`"""
def build_lib_path(this_script=__file__):
"""Return path to the common build library directory"""
hacking_dir = os.path.dirname(this_script)
libdir = os.path.abspath(os.path.join(hacking_dir, 'build_library'))
if libdir not in sys.path:
sys.path.insert(0, libdir)
return libdir
set_sys_path()
sys.path.insert(0, build_lib_path())
from build_ansible import commands
from build_ansible import commands, errors
def create_arg_parser(program_name):
@ -63,13 +62,26 @@ def main():
argcomplete.autocomplete(arg_parser)
args = arg_parser.parse_args(sys.argv[1:])
if args.command is None:
print('Please specify a subcommand to run')
sys.exit(1)
for subcommand in subcommands:
if subcommand.name == args.command:
sys.exit(subcommand.main(args))
command = subcommand
break
else:
# Note: We should never trigger this because argparse should shield us from it
print('Error: {0} was not a recognized subcommand'.format(args.command))
sys.exit(1)
print('Error: Select a subcommand')
arg_parser.print_usage()
try:
retval = command.main(args)
except errors.DependencyError as e:
print(e)
sys.exit(2)
sys.exit(retval)
if __name__ == '__main__':

0
lib/ansible/utils/_build_helpers.py → hacking/build_library/build_ansible/change_detection.py
View file

80
hacking/build_library/build_ansible/command_plugins/dump_config.py Normal file
View file

@ -0,0 +1,80 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import os.path
import pathlib
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
DEFAULT_TEMPLATE_FILE = 'config.rst.j2'
DEFAULT_TEMPLATE_DIR = pathlib.Path(__file__).parents[4] / 'docs/templates'
def fix_description(config_options):
'''some descriptions are strings, some are lists. workaround it...'''
for config_key in config_options:
description = config_options[config_key].get('description', [])
if isinstance(description, list):
desc_list = description
else:
desc_list = [description]
config_options[config_key]['description'] = desc_list
return config_options
class DocumentConfig(Command):
name = 'document-config'
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(cls.name, description='Generate module documentation from metadata')
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
default=DEFAULT_TEMPLATE_FILE,
help="Jinja2 template to use for the config")
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
default=DEFAULT_TEMPLATE_DIR,
help="directory containing Jinja2 templates")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/',
help="Output directory for rst files")
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
help="Source for attribute docs")
@staticmethod
def main(args):
output_dir = os.path.abspath(args.output_dir)
template_file_full_path = os.path.abspath(os.path.join(args.template_dir, args.template_file))
template_file = os.path.basename(template_file_full_path)
template_dir = os.path.dirname(template_file_full_path)
if args.docs:
with open(args.docs) as f:
docs = yaml.safe_load(f)
else:
docs = {}
config_options = docs
config_options = fix_description(config_options)
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(template_file)
output_name = os.path.join(output_dir, template_file.replace('.j2', ''))
temp_vars = {'config_options': config_options}
data = to_bytes(template.render(temp_vars))
update_file_if_different(output_name, data)
return 0

125
hacking/build_library/build_ansible/command_plugins/dump_keywords.py Normal file
View file

@ -0,0 +1,125 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import importlib
import os.path
import pathlib
import re
from distutils.version import LooseVersion
import jinja2
import yaml
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
DEFAULT_TEMPLATE_DIR = str(pathlib.Path(__file__).resolve().parents[4] / 'docs/templates')
TEMPLATE_FILE = 'playbooks_keywords.rst.j2'
PLAYBOOK_CLASS_NAMES = ['Play', 'Role', 'Block', 'Task']
def load_definitions(keyword_definitions_file):
docs = {}
with open(keyword_definitions_file) as f:
docs = yaml.safe_load(f)
return docs
def extract_keywords(keyword_definitions):
pb_keywords = {}
for pb_class_name in PLAYBOOK_CLASS_NAMES:
if pb_class_name == 'Play':
module_name = 'ansible.playbook'
else:
module_name = 'ansible.playbook.{0}'.format(pb_class_name.lower())
module = importlib.import_module(module_name)
playbook_class = getattr(module, pb_class_name, None)
if playbook_class is None:
raise ImportError("We weren't able to import the module {0}".format(module_name))
# Maintain order of the actual class names for our output
# Build up a mapping of playbook classes to the attributes that they hold
pb_keywords[pb_class_name] = {k: v for (k, v) in playbook_class._valid_attrs.items()
# Filter private attributes as they're not usable in playbooks
if not v.private}
# pick up definitions if they exist
for keyword in tuple(pb_keywords[pb_class_name]):
if keyword in keyword_definitions:
pb_keywords[pb_class_name][keyword] = keyword_definitions[keyword]
else:
# check if there is an alias, otherwise undocumented
alias = getattr(getattr(playbook_class, '_%s' % keyword), 'alias', None)
if alias and alias in keyword_definitions:
pb_keywords[pb_class_name][alias] = keyword_definitions[alias]
del pb_keywords[pb_class_name][keyword]
else:
pb_keywords[pb_class_name][keyword] = ' UNDOCUMENTED!! '
# loop is really with_ for users
if pb_class_name == 'Task':
pb_keywords[pb_class_name]['with_<lookup_plugin>'] = (
'The same as ``loop`` but magically adds the output of any lookup plugin to'
' generate the item list.')
# local_action is implicit with action
if 'action' in pb_keywords[pb_class_name]:
pb_keywords[pb_class_name]['local_action'] = ('Same as action but also implies'
' ``delegate_to: localhost``')
return pb_keywords
def generate_page(pb_keywords, template_dir):
env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True,)
template = env.get_template(TEMPLATE_FILE)
tempvars = {'pb_keywords': pb_keywords, 'playbook_class_names': PLAYBOOK_CLASS_NAMES}
keyword_page = template.render(tempvars)
if LooseVersion(jinja2.__version__) < LooseVersion('2.10'):
# jinja2 < 2.10's indent filter indents blank lines. Cleanup
keyword_page = re.sub(' +\n', '\n', keyword_page)
return keyword_page
class DocumentKeywords(Command):
name = 'document-keywords'
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(cls.name, description='Generate playbook keyword documentation from'
' code and descriptions')
parser.add_argument("-T", "--template-dir", action="store", dest="template_dir",
default=DEFAULT_TEMPLATE_DIR,
help="directory containing Jinja2 templates")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
default='/tmp/', help="Output directory for rst files")
parser.add_argument("-d", "--docs-source", action="store", dest="docs", default=None,
help="Source for attribute docs")
@staticmethod
def main(args):
if not args.docs:
print('Definitions for keywords must be specified via `--docs-source FILENAME`')
return 1
keyword_definitions = load_definitions(args.docs)
pb_keywords = extract_keywords(keyword_definitions)
keyword_page = generate_page(pb_keywords, args.template_dir)
outputname = os.path.join(args.output_dir, TEMPLATE_FILE.replace('.j2', ''))
update_file_if_different(outputname, to_bytes(keyword_page))
return 0

153
docs/bin/generate_man.py → hacking/build_library/build_ansible/command_plugins/generate_man.py Executable file → Normal file
View file

@ -1,25 +1,27 @@
#!/usr/bin/env python
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import argparse
import os
import os.path
import pathlib
import sys
from jinja2 import Environment, FileSystemLoader
from ansible.module_utils._text import to_bytes
from ansible.utils._build_helpers import update_file_if_different
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
def generate_parser():
p = argparse.ArgumentParser(
description='Generate cli documentation from cli docstrings',
)
p.add_argument("-t", "--template-file", action="store", dest="template_file", default="../templates/man.j2", help="path to jinja2 template")
p.add_argument("-o", "--output-dir", action="store", dest="output_dir", default='/tmp/', help="Output directory for rst files")
p.add_argument("-f", "--output-format", action="store", dest="output_format", default='man', help="Output format for docs (the default 'man' or 'rst')")
p.add_argument('args', help='CLI module(s)', metavar='module', nargs='*')
return p
DEFAULT_TEMPLATE_FILE = pathlib.Path(__file__).parents[4] / 'docs/templates/man.j2'
# from https://www.python.org/dev/peps/pep-0257/
@ -213,78 +215,89 @@ def opts_docs(cli_class_name, cli_module_name):
return docs
if __name__ == '__main__':
class GenerateMan(Command):
name = 'generate-man'
parser = generate_parser()
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(name=cls.name,
description='Generate cli documentation from cli docstrings')
options = parser.parse_args()
parser.add_argument("-t", "--template-file", action="store", dest="template_file",
default=DEFAULT_TEMPLATE_FILE, help="path to jinja2 template")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
default='/tmp/', help="Output directory for rst files")
parser.add_argument("-f", "--output-format", action="store", dest="output_format",
default='man',
help="Output format for docs (the default 'man' or 'rst')")
parser.add_argument('cli_modules', help='CLI module name(s)', metavar='MODULE_NAME', nargs='*')
template_file = options.template_file
template_path = os.path.expanduser(template_file)
template_dir = os.path.abspath(os.path.dirname(template_path))
template_basename = os.path.basename(template_file)
@staticmethod
def main(args):
template_file = args.template_file
template_path = os.path.expanduser(template_file)
template_dir = os.path.abspath(os.path.dirname(template_path))
template_basename = os.path.basename(template_file)
output_dir = os.path.abspath(options.output_dir)
output_format = options.output_format
output_dir = os.path.abspath(args.output_dir)
output_format = args.output_format
cli_modules = options.args
cli_modules = args.cli_modules
# various cli parsing things checks sys.argv if the 'args' that are passed in are []
# so just remove any args so the cli modules dont try to parse them resulting in warnings
sys.argv = [sys.argv[0]]
# need to be in right dir
os.chdir(os.path.dirname(__file__))
# various cli parsing things checks sys.argv if the 'args' that are passed in are []
# so just remove any args so the cli modules dont try to parse them resulting in warnings
sys.argv = [sys.argv[0]]
allvars = {}
output = {}
cli_list = []
cli_bin_name_list = []
allvars = {}
output = {}
cli_list = []
cli_bin_name_list = []
# for binary in os.listdir('../../lib/ansible/cli'):
for cli_module_name in cli_modules:
binary = os.path.basename(os.path.expanduser(cli_module_name))
# for binary in os.listdir('../../lib/ansible/cli'):
for cli_module_name in cli_modules:
binary = os.path.basename(os.path.expanduser(cli_module_name))
if not binary.endswith('.py'):
continue
elif binary == '__init__.py':
continue
if not binary.endswith('.py'):
continue
elif binary == '__init__.py':
continue
cli_name = os.path.splitext(binary)[0]
cli_name = os.path.splitext(binary)[0]
if cli_name == 'adhoc':
cli_class_name = 'AdHocCLI'
# myclass = 'AdHocCLI'
output[cli_name] = 'ansible.1.rst.in'
cli_bin_name = 'ansible'
else:
# myclass = "%sCLI" % libname.capitalize()
cli_class_name = "%sCLI" % cli_name.capitalize()
output[cli_name] = 'ansible-%s.1.rst.in' % cli_name
cli_bin_name = 'ansible-%s' % cli_name
if cli_name == 'adhoc':
cli_class_name = 'AdHocCLI'
# myclass = 'AdHocCLI'
output[cli_name] = 'ansible.1.rst.in'
cli_bin_name = 'ansible'
else:
# myclass = "%sCLI" % libname.capitalize()
cli_class_name = "%sCLI" % cli_name.capitalize()
output[cli_name] = 'ansible-%s.1.rst.in' % cli_name
cli_bin_name = 'ansible-%s' % cli_name
# FIXME:
allvars[cli_name] = opts_docs(cli_class_name, cli_name)
cli_bin_name_list.append(cli_bin_name)
# FIXME:
allvars[cli_name] = opts_docs(cli_class_name, cli_name)
cli_bin_name_list.append(cli_bin_name)
cli_list = allvars.keys()
cli_list = allvars.keys()
doc_name_formats = {'man': '%s.1.rst.in',
'rst': '%s.rst'}
doc_name_formats = {'man': '%s.1.rst.in',
'rst': '%s.rst'}
for cli_name in cli_list:
for cli_name in cli_list:
# template it!
env = Environment(loader=FileSystemLoader(template_dir))
template = env.get_template(template_basename)
# template it!
env = Environment(loader=FileSystemLoader(template_dir))
template = env.get_template(template_basename)
# add rest to vars
tvars = allvars[cli_name]
tvars['cli_list'] = cli_list
tvars['cli_bin_name_list'] = cli_bin_name_list
tvars['cli'] = cli_name
if '-i' in tvars['options']:
print('uses inventory')
# add rest to vars
tvars = allvars[cli_name]
tvars['cli_list'] = cli_list
tvars['cli_bin_name_list'] = cli_bin_name_list
tvars['cli'] = cli_name
if '-i' in tvars['options']:
print('uses inventory')
manpage = template.render(tvars)
filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
update_file_if_different(filename, to_bytes(manpage))
manpage = template.render(tvars)
filename = os.path.join(output_dir, doc_name_formats[output_format] % tvars['cli_name'])
update_file_if_different(filename, to_bytes(manpage))

177
docs/bin/plugin_formatter.py → hacking/build_library/build_ansible/command_plugins/plugin_formatter.py Executable file → Normal file
View file

@ -1,4 +1,3 @@
#!/usr/bin/env python
# Copyright: (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# Copyright: (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
# Copyright: (c) 2017, Ansible Project
@ -45,7 +44,10 @@ from ansible.module_utils.six import iteritems, string_types
from ansible.plugins.loader import fragment_loader
from ansible.utils import plugin_docs
from ansible.utils.display import Display
from ansible.utils._build_helpers import update_file_if_different
# Pylint doesn't understand Python3 namespace modules.
from ..change_detection import update_file_if_different # pylint: disable=relative-beyond-top-level
from ..commands import Command # pylint: disable=relative-beyond-top-level
#####################################################################################
@ -363,29 +365,6 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
return module_info, categories
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2',
description='Generate module documentation from metadata',
)
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-P", "--plugin-type", action="store", dest="plugin_type", default='module', help="The type of plugin (module, lookup, etc)")
p.add_option("-T", "--template-dir", action="append", dest="template_dir", help="directory containing Jinja2 templates")
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
p.add_option("-l", "--limit-to-modules", '--limit-to', action="store", dest="limit_to", default=None,
help="Limit building module documentation to comma-separated list of plugins. Specify non-existing plugin name for no plugins.")
p.add_option('-V', action='version', help='Show version number and exit')
p.add_option('-v', '--verbose', dest='verbosity', default=0, action="count", help="verbose mode (increase number of 'v's for more)")
return p
def jinja2_environment(template_dir, typ, plugin_type):
env = Environment(loader=FileSystemLoader(template_dir),
@ -734,82 +713,106 @@ def validate_options(options):
sys.exit("--template-dir must be specified")
def main():
class DocumentPlugins(Command):
name = 'document-plugins'
# INIT
p = generate_parser()
(options, args) = p.parse_args()
if not options.template_dir:
options.template_dir = ["hacking/templates"]
validate_options(options)
display.verbosity = options.verbosity
plugin_type = options.plugin_type
@classmethod
def init_parser(cls, add_parser):
parser = add_parser(cls.name, description='Generate module documentation from metadata')
display.display("Evaluating %s files..." % plugin_type)
parser.add_argument("-A", "--ansible-version", action="store", dest="ansible_version",
default="unknown", help="Ansible version number")
parser.add_argument("-M", "--module-dir", action="store", dest="module_dir",
default=MODULEDIR, help="Ansible library path")
parser.add_argument("-P", "--plugin-type", action="store", dest="plugin_type",
default='module', help="The type of plugin (module, lookup, etc)")
parser.add_argument("-T", "--template-dir", action="append", dest="template_dir",
help="directory containing Jinja2 templates")
parser.add_argument("-t", "--type", action='store', dest='type', choices=['rst'],
default='rst', help="Document type")
parser.add_argument("-o", "--output-dir", action="store", dest="output_dir", default=None,
help="Output directory for module files")
parser.add_argument("-I", "--includes-file", action="store", dest="includes_file",
default=None, help="Create a file containing list of processed modules")
parser.add_argument("-l", "--limit-to-modules", '--limit-to', action="store",
dest="limit_to", default=None, help="Limit building module documentation"
" to comma-separated list of plugins. Specify non-existing plugin name"
" for no plugins.")
parser.add_argument('-V', action='version', help='Show version number and exit')
parser.add_argument('-v', '--verbose', dest='verbosity', default=0, action="count",
help="verbose mode (increase number of 'v's for more)")
# prep templating
templates = jinja2_environment(options.template_dir, options.type, plugin_type)
@staticmethod
def main(args):
if not args.template_dir:
args.template_dir = ["hacking/templates"]
validate_options(args)
display.verbosity = args.verbosity
plugin_type = args.plugin_type
# set file/directory structure
if plugin_type == 'module':
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
outputname = '%s_' + '%s.rst' % plugin_type
output_dir = options.output_dir
else:
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
outputname = '%s.rst'
output_dir = '%s/plugins/%s' % (options.output_dir, plugin_type)
display.display("Evaluating %s files..." % plugin_type)
display.vv('output name: %s' % outputname)
display.vv('output dir: %s' % output_dir)
# prep templating
templates = jinja2_environment(args.template_dir, args.type, plugin_type)
# Convert passed-in limit_to to None or list of modules.
if options.limit_to is not None:
options.limit_to = [s.lower() for s in options.limit_to.split(",")]
# set file/directory structure
if plugin_type == 'module':
# trim trailing s off of plugin_type for plugin_type=='modules'. ie 'copy_module.rst'
outputname = '%s_' + '%s.rst' % plugin_type
output_dir = args.output_dir
else:
# for plugins, just use 'ssh.rst' vs 'ssh_module.rst'
outputname = '%s.rst'
output_dir = '%s/plugins/%s' % (args.output_dir, plugin_type)
plugin_info, categories = get_plugin_info(options.module_dir, limit_to=options.limit_to, verbose=(options.verbosity > 0))
display.vv('output name: %s' % outputname)
display.vv('output dir: %s' % output_dir)
categories['all'] = {'_modules': plugin_info.keys()}
# Convert passed-in limit_to to None or list of modules.
if args.limit_to is not None:
args.limit_to = [s.lower() for s in args.limit_to.split(",")]
if display.verbosity >= 3:
display.vvv(pp.pformat(categories))
if display.verbosity >= 5:
display.vvvvv(pp.pformat(plugin_info))
plugin_info, categories = get_plugin_info(args.module_dir, limit_to=args.limit_to, verbose=(args.verbosity > 0))
# Transform the data
if options.type == 'rst':
display.v('Generating rst')
for key, record in plugin_info.items():
display.vv(key)
if display.verbosity >= 5:
display.vvvvv(pp.pformat(('record', record)))
if record.get('doc', None):
short_desc = record['doc']['short_description'].rstrip('.')
if short_desc is None:
display.warning('short_description for %s is None' % key)
short_desc = ''
record['doc']['short_description'] = rst_ify(short_desc)
categories['all'] = {'_modules': plugin_info.keys()}
if plugin_type == 'module':
display.v('Generating Categories')
# Write module master category list
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
category_index_name = '%ss_by_category.rst' % plugin_type
write_data(category_list_text, output_dir, category_index_name)
if display.verbosity >= 3:
display.vvv(pp.pformat(categories))
if display.verbosity >= 5:
display.vvvvv(pp.pformat(plugin_info))
# Render all the individual plugin pages
display.v('Generating plugin pages')
process_plugins(plugin_info, templates, outputname, output_dir, options.ansible_version, plugin_type)
# Transform the data
if args.type == 'rst':
display.v('Generating rst')
for key, record in plugin_info.items():
display.vv(key)
if display.verbosity >= 5:
display.vvvvv(pp.pformat(('record', record)))
if record.get('doc', None):
short_desc = record['doc']['short_description'].rstrip('.')
if short_desc is None:
display.warning('short_description for %s is None' % key)
short_desc = ''
record['doc']['short_description'] = rst_ify(short_desc)
# Render all the categories for modules
if plugin_type == 'module':
display.v('Generating Category lists')
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
if plugin_type == 'module':
display.v('Generating Categories')
# Write module master category list
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
category_index_name = '%ss_by_category.rst' % plugin_type
write_data(category_list_text, output_dir, category_index_name)
# Render all the individual plugin pages
display.v('Generating plugin pages')
process_plugins(plugin_info, templates, outputname, output_dir, args.ansible_version, plugin_type)
# Render all the categories for modules
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
if plugin_type == 'module':
display.v('Generating Category lists')
category_list_name_template = 'list_of_%s_' + '%ss.rst' % plugin_type
process_categories(plugin_info, categories, templates, output_dir, category_list_name_template, plugin_type)
# Render all the categories for modules
process_support_levels(plugin_info, categories, templates, output_dir, plugin_type)
if __name__ == '__main__':
main()
return 0

2
hacking/build_library/build_ansible/command_plugins/porting_guide.py
View file

@ -121,7 +121,7 @@ def generate_porting_guide(version):
def write_guide(version, guide_content):
filename = f'porting_guide_{version}.rst'
filename = 'porting_guide_{0}.rst'.format(version)
with open(filename, 'w') as out_file:
out_file.write(guide_content)

256
hacking/build_library/build_ansible/command_plugins/release_announcement.py
View file

@ -7,148 +7,13 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import argparse
import asyncio
import datetime
import hashlib
import os.path
import sys
from collections import UserString
from distutils.version import LooseVersion
import aiohttp
from jinja2 import Environment, DictLoader
# Pylint doesn't understand Python3 namespace modules.
from ..commands import Command # pylint: disable=relative-beyond-top-level
# pylint: disable=
VERSION_FRAGMENT = """
{%- if versions | length > 1 %}
{% for version in versions %}
{% if loop.last %}and {{ version }}{% else %}
{% if versions | length == 2 %}{{ version }} {% else %}{{ version }}, {% endif -%}
{% endif -%}
{% endfor -%}
{%- else %}{{ versions[0] }}{% endif -%}
"""
LONG_TEMPLATE = """
{% set plural = False if versions | length == 1 else True %}
{% set latest_ver = (versions | sort(attribute='ver_obj'))[-1] %}
To: ansible-devel@googlegroups.com, ansible-project@googlegroups.com, ansible-announce@googlegroups.com
Subject: New Ansible release{% if plural %}s{% endif %} {{ version_str }}
{% filter wordwrap %}
Hi all- we're happy to announce that the general release of Ansible {{ version_str }}{% if plural %} are{%- else %} is{%- endif %} now available!
{% endfilter %}
How do you get it?
------------------
{% for version in versions %}
$ pip install ansible=={{ version }} --user
{% if not loop.last %}
or
{% endif %}
{% endfor %}
The tar.gz of the release{% if plural %}s{% endif %} can be found here:
{% for version in versions %}
* {{ version }}
https://releases.ansible.com/ansible/ansible-{{ version }}.tar.gz
SHA256: {{ hashes[version] }}
{% endfor %}
What's new in {{ version_str }}
{{ '-' * (14 + version_str | length) }}
{% filter wordwrap %}
{% if plural %}These releases are{% else %}This release is a{% endif %} maintenance release{% if plural %}s{% endif %} containing numerous bugfixes. The full {% if plural %} changelogs are{% else %} changelog is{% endif %} at:
{% endfilter %}
{% for version in versions %}
* {{ version }}
https://github.com/ansible/ansible/blob/stable-{{ version.split('.')[:2] | join('.') }}/changelogs/CHANGELOG-v{{ version.split('.')[:2] | join('.') }}.rst
{% endfor %}
What's the schedule for future maintenance releases?
----------------------------------------------------
{% filter wordwrap %}
Future maintenance releases will occur approximately every 3 weeks. So expect the next one around {{ next_release.strftime('%Y-%m-%d') }}.
{% endfilter %}
Porting Help
------------
{% filter wordwrap %}
We've published a porting guide at
https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_{{ latest_ver.split('.')[:2] | join('.') }}.html to help migrate your content to {{ latest_ver.split('.')[:2] | join('.') }}.
{% endfilter %}
{% filter wordwrap %}
If you discover any errors or if any of your working playbooks break when you upgrade to {{ latest_ver }}, please use the following link to report the regression:
{% endfilter %}
https://github.com/ansible/ansible/issues/new/choose
{% filter wordwrap %}
In your issue, be sure to mention the Ansible version that works and the one that doesn't.
{% endfilter %}
Thanks!
-{{ name }}
""" # noqa for E501 (line length).
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
# proper wrapping to occur
SHORT_TEMPLATE = """
{% set plural = False if versions | length == 1 else True %}
@ansible
{{ version_str }}
{% if plural %}
have
{% else %}
has
{% endif %}
been released! Get
{% if plural %}
them
{% else %}
it
{% endif %}
on PyPI: pip install ansible=={{ (versions|sort(attribute='ver_obj'))[-1] }},
https://releases.ansible.com/ansible/, the Ansible PPA on Launchpad, or GitHub. Happy automating!
""" # noqa for E501 (line length).
# jinja2 is horrid about getting rid of extra newlines so we have to have a single per paragraph for
# proper wrapping to occur
JINJA_ENV = Environment(
loader=DictLoader({'long': LONG_TEMPLATE,
'short': SHORT_TEMPLATE,
'version_string': VERSION_FRAGMENT,
}),
extensions=['jinja2.ext.i18n'],
trim_blocks=True,
lstrip_blocks=True,
)
from .. import errors # pylint: disable=relative-beyond-top-level
class VersionStr(UserString):
@ -167,108 +32,6 @@ def transform_args(args):
return args
async def calculate_hash_from_tarball(session, version):
tar_url = f'https://releases.ansible.com/ansible/ansible-{version}.tar.gz'
tar_task = asyncio.create_task(session.get(tar_url))
tar_response = await tar_task
tar_hash = hashlib.sha256()
while True:
chunk = await tar_response.content.read(1024)
if not chunk:
break
tar_hash.update(chunk)
return tar_hash.hexdigest()
async def parse_hash_from_file(session, version):
filename = f'ansible-{version}.tar.gz'
hash_url = f'https://releases.ansible.com/ansible/{filename}.sha'
hash_task = asyncio.create_task(session.get(hash_url))
hash_response = await hash_task
hash_content = await hash_response.read()
precreated_hash, precreated_filename = hash_content.split(None, 1)
if filename != precreated_filename.strip().decode('utf-8'):
raise ValueError(f'Hash file contains hash for a different file: {precreated_filename}')
return precreated_hash.decode('utf-8')
async def get_hash(session, version):
calculated_hash = await calculate_hash_from_tarball(session, version)
precreated_hash = await parse_hash_from_file(session, version)
if calculated_hash != precreated_hash:
raise ValueError(f'Hash in file ansible-{version}.tar.gz.sha {precreated_hash} does not'
f' match hash of tarball {calculated_hash}')
return calculated_hash
async def get_hashes(versions):
hashes = {}
requestors = {}
async with aiohttp.ClientSession() as aio_session:
for version in versions:
requestors[version] = asyncio.create_task(get_hash(aio_session, version))
for version, request in requestors.items():
await request
hashes[version] = request.result()
return hashes
def next_release_date(weeks=3):
days_in_the_future = weeks * 7
today = datetime.datetime.now()
numeric_today = today.weekday()
# We release on Thursdays
if numeric_today == 3:
# 3 is Thursday
pass
elif numeric_today == 4:
# If this is Friday, we can adjust back to Thursday for the next release
today -= datetime.timedelta(days=1)
elif numeric_today < 3:
# Otherwise, slide forward to Thursday
today += datetime.timedelta(days=(3 - numeric_today))
else:
# slightly different formula if it's past Thursday this week. We need to go forward to
# Thursday of next week
today += datetime.timedelta(days=(10 - numeric_today))
next_release = today + datetime.timedelta(days=days_in_the_future)
return next_release
def generate_long_message(versions, name):
hashes = asyncio.run(get_hashes(versions))
version_template = JINJA_ENV.get_template('version_string')
version_str = version_template.render(versions=versions).strip()
next_release = next_release_date()
template = JINJA_ENV.get_template('long')
message = template.render(versions=versions, version_str=version_str,
name=name, hashes=hashes, next_release=next_release)
return message
def generate_short_message(versions):
version_template = JINJA_ENV.get_template('version_string')
version_str = version_template.render(versions=versions).strip()
template = JINJA_ENV.get_template('short')
message = template.render(versions=versions, version_str=version_str)
message = ' '.join(message.split()) + '\n'
return message
def write_message(filename, message):
if filename != '-':
with open(filename, 'w') as out_file:
@ -294,12 +57,21 @@ class ReleaseAnnouncementCommand(Command):
parser.add_argument("--twitter-out", type=str, default="-",
help="Filename to place the twitter announcement into")
@staticmethod
def main(args):
@classmethod
def main(cls, args):
if sys.version_info < (3, 6):
raise errors.DependencyError('The {0} subcommand needs Python-3.6+'
' to run'.format(cls.name))
# Import here because these functions are invalid on Python-3.5 and the command plugins and
# init_parser() method need to be compatible with Python-3.4+ for now.
# Pylint doesn't understand Python3 namespace modules.
from .. announce import create_short_message, create_long_message # pylint: disable=relative-beyond-top-level
args = transform_args(args)
twitter_message = generate_short_message(args.versions)
email_message = generate_long_message(args.versions, args.name)
twitter_message = create_short_message(args.versions)
email_message = create_long_message(args.versions, args.name)
write_message(args.twitter_out, twitter_message)
write_message(args.email_out, email_message)

12
hacking/build_library/build_ansible/errors.py Normal file
View file

@ -0,0 +1,12 @@
# coding: utf-8
# Copyright: (c) 2019, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
class DependencyError(Exception):
"""Used when a dependency is unmet"""
pass

5
lib/ansible/playbook/attribute.py
View file

@ -52,8 +52,9 @@ class Attribute:
:kwarg isa: The type of the attribute. Allowable values are a string
representation of any yaml basic datatype, python class, or percent.
(Enforced at post-validation time).
:kwarg private: Hides the attribute from being documented.
TODO: it should also should prevent it from being user settable
:kwarg private: Not used at runtime. The docs playbook keyword dumper uses it to determine
that a keyword should not be documented. mpdehaan had plans to remove attributes marked
private from the ds so they would not have been available at all.
:kwarg default: Default value if unspecified in the YAML document.
:kwarg required: Whether or not the YAML document must contain this field.
If the attribute is None when post-validated, an error will be raised.

8
test/sanity/compile/python2.6-skip.txt
View file

@ -1,3 +1,11 @@
# The following are only run by release engineers who can be asked to have newer Python3 on their systems
hacking/build_library/build_ansible/command_plugins/porting_guide.py
hacking/build_library/build_ansible/command_plugins/release_announcement.py
# The following are used to build docs. Since we explicitly say that the controller won't run on
# Python-2.6 (docs are built controller-side) and EPEL-6, the only LTS platform with Python-2.6,
# doesn't have a new enough sphinx to build docs, do not test these under Python-2.6
hacking/build_library/build_ansible/command_plugins/dump_config.py
hacking/build_library/build_ansible/command_plugins/dump_keywords.py
hacking/build_library/build_ansible/command_plugins/generate_man.py
hacking/build_library/build_ansible/command_plugins/plugin_formatter.py