cmueller/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
ansible/docs/bin/plugin_formatter.py
Toshio Kuratomi 546187a8af Revamp the plugin_formatter doc generator 
* Use a template to generate the category lists
* Refactor so that we first extract all of the data that we need to
  build the docs and then give that data to the templates to build with
* Add docs page listing modules ordered by support level
2017-08-15 23:12:08 -07:00

497 lines
19 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/env python
# (c) 2012, Jan-Piet Mens <jpmens () gmail.com>
# (c) 2012-2014, Michael DeHaan <michael@ansible.com> and others
# (c) 2017 Ansible Project
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import, division, print_function
__metaclass__ = type
import datetime
import glob
import optparse
import os
import re
import sys
import warnings
from collections import defaultdict
try:
from html import escape as html_escape
except ImportError:
# Python-3.2 or later
import cgi
def html_escape(text, quote=True):
return cgi.escape(text, quote)
import yaml
from jinja2 import Environment, FileSystemLoader
from six import iteritems
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes
from ansible.utils import plugin_docs
#####################################################################################
# constants and paths
# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.3
# Get parent directory of the directory this script lives in
MODULEDIR = os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'lib', 'ansible', 'modules'
))
# The name of the DOCUMENTATION template
EXAMPLE_YAML = os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
))
_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL = re.compile(r"U\(([^)]+)\)")
_CONST = re.compile(r"C\(([^)]+)\)")
DEPRECATED = b" (D)"
def rst_ify(text):
''' convert symbols like I(this is in italics) to valid restructured text '''
try:
t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
t = _BOLD.sub(r'**' + r"\1" + r"**", t)
t = _MODULE.sub(r':ref:`' + r"\1 <\1>" + r"`", t)
t = _URL.sub(r"\1", t)
t = _CONST.sub(r'``' + r"\1" + r"``", t)
except Exception as e:
raise AnsibleError("Could not process (%s) : %s" % (str(text), str(e)))
return t
def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML '''
t = html_escape(text)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def rst_fmt(text, fmt):
''' helper for Jinja2 to do format strings '''
return fmt % (text)
def rst_xline(width, char="="):
''' return a restructured text line of a given length '''
return char * width
def write_data(text, output_dir, outputname, module=None):
''' dumps module output to a file or the screen, as requested '''
if output_dir is not None:
if module:
outputname = outputname % module
fname = os.path.join(output_dir, outputname)
fname = fname.replace(".py", "")
with open(fname, 'wb') as f:
f.write(to_bytes(text))
else:
print(text)
def get_module_info(module_dir, limit_to_modules=None, verbose=False):
'''
Returns information about modules and the categories that they belong to
:arg module_dir: file system path to the top of the modules directory
:kwarg limit_to_modules: If given, this is a list of module names to
generate information for. All other modules will be ignored.
:returns: Tuple of two dicts containing module_info, categories, and
aliases and a set listing deprecated modules:
:module_info: mapping of module names to information about them. The fields of the dict are:
:path: filesystem path to the module
:deprecated: boolean. True means the module is deprecated otherwise not.
:aliases: set of aliases to this module name
:metadata: The modules metadata (as recorded in the module)
:doc: The documentation structure for the module
:examples: The module's examples
:returndocs: The module's returndocs
:categories: maps category names to a dict. The dict contains at
least one key, '_modules' which contains a list of module names in
that category. Any other keys in the dict are subcategories with
the same structure.
'''
categories = dict()
module_info = defaultdict(dict)
# * windows powershell modules have documentation stubs in python docstring
# format (they are not executed) so skip the ps1 format files
# * One glob level for every module level that we're going to traverse
files = (
glob.glob("%s/*.py" % module_dir) +
glob.glob("%s/*/*.py" % module_dir) +
glob.glob("%s/*/*/*.py" % module_dir) +
glob.glob("%s/*/*/*/*.py" % module_dir)
)
for module_path in files:
# Do not list __init__.py files
if module_path.endswith('__init__.py'):
continue
# Do not list blacklisted modules
module = os.path.splitext(os.path.basename(module_path))[0]
if module in plugin_docs.BLACKLIST['MODULE']:
continue
# If requested, limit module documentation building only to passed-in
# modules.
if limit_to_modules is not None and module.lower() in limit_to_modules:
continue
deprecated = False
if module.startswith("_"):
if os.path.islink(module_path):
# Handle aliases
source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
module = module.replace("_", "", 1)
aliases = module_info[source].get('aliases', set())
aliases.add(module)
# In case we just created this via get()'s fallback
module_info[source]['aliases'] = aliases
continue
else:
# Handle deprecations
module = module.replace("_", "", 1)
deprecated = True
#
# Regular module to process
#
category = categories
# Start at the second directory because we don't want the "vendor"
mod_path_only = os.path.dirname(module_path[len(module_dir):])
# build up the categories that this module belongs to
for new_cat in mod_path_only.split('/')[1:]:
if new_cat not in category:
category[new_cat] = dict()
category[new_cat]['_modules'] = []
category = category[new_cat]
category['_modules'].append(module)
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, verbose=verbose)
# save all the information
module_info[module] = {'path': module_path,
'deprecated': deprecated,
'aliases': set(),
'metadata': metadata,
'doc': doc,
'examples': examples,
'returndocs': returndocs,
}
# keep module tests out of becoming module docs
if 'test' in categories:
del categories['test']
return module_info, categories
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2',
description='Generate module documentation from metadata',
)
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
p.add_option("-t", "--type", action='store', dest='type', choices=['rst'], default='rst', help="Document type")
p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
p.add_option("-l", "--limit-to-modules", action="store", dest="limit_to_modules", default=None,
help="Limit building module documentation to comma-separated list of modules. Specify non-existing module name for no modules.")
p.add_option('-V', action='version', help='Show version number and exit')
return p
def jinja2_environment(template_dir, typ):
env = Environment(loader=FileSystemLoader(template_dir),
variable_start_string="@{",
variable_end_string="}@",
trim_blocks=True)
env.globals['xline'] = rst_xline
templates = {}
if typ == 'rst':
env.filters['convert_symbols_to_format'] = rst_ify
env.filters['html_ify'] = html_ify
env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline
templates['plugin'] = env.get_template('plugin.rst.j2')
templates['category_list'] = env.get_template('modules_by_category.rst.j2')
templates['support_list'] = env.get_template('modules_by_support.rst.j2')
templates['list_of_CATEGORY_modules'] = env.get_template('list_of_CATEGORY_modules.rst.j2')
outputname = "%s_module.rst"
else:
raise Exception("unknown module format type: %s" % typ)
return templates, outputname
def too_old(added):
if not added:
return False
try:
added_tokens = str(added).split(".")
readded = added_tokens[0] + "." + added_tokens[1]
added_float = float(readded)
except ValueError as e:
warnings.warn("Could not parse %s: %s" % (added, str(e)))
return False
return added_float < TO_OLD_TO_BE_NOTABLE
def process_modules(module_map, templates, outputname, output_dir, ansible_version):
for module in module_map:
print("rendering: %s" % module)
fname = module_map[module]['path']
# crash if module is missing documentation and not explicitly hidden from docs index
if module_map[module]['doc'] is None:
sys.exit("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
# Going to reference this heavily so make a short name to reference it by
doc = module_map[module]['doc']
if module_map[module]['deprecated'] and 'deprecated' not in doc:
sys.exit("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
if 'version_added' not in doc:
sys.exit("*** ERROR: missing version_added in: %s ***\n" % module)
#
# The present template gets everything from doc so we spend most of this
# function moving data into doc for the template to reference
#
if module_map[module]['aliases']:
doc['aliases'] = module_map[module]['aliases']
# don't show version added information if it's too old to be called out
added = 0
if doc['version_added'] == 'historical':
del doc['version_added']
else:
added = doc['version_added']
# Strip old version_added for the module
if too_old(added):
del doc['version_added']
option_names = []
if 'options' in doc and doc['options']:
for (k, v) in iteritems(doc['options']):
# Error out if there's no description
if 'description' not in doc['options'][k]:
raise AnsibleError("Missing required description for option %s in %s " % (k, module))
# Error out if required isn't a boolean (people have been putting
# information on when something is required in here. Those need
# to go in the description instead).
required_value = doc['options'][k].get('required', False)
if not isinstance(required_value, bool):
raise AnsibleError("Invalid required value '%s' for option '%s' in '%s' (must be truthy)" % (required_value, k, module))
# Strip old version_added information for options
if 'version_added' in doc['options'][k] and too_old(doc['options'][k]['version_added']):
del doc['options'][k]['version_added']
# Make sure description is a list of lines for later formatting
if not isinstance(doc['options'][k]['description'], list):
doc['options'][k]['description'] = [doc['options'][k]['description']]
option_names.append(k)
option_names.sort()
doc['option_keys'] = option_names
doc['filename'] = fname
doc['docuri'] = doc['module'].replace('_', '-')
doc['now_date'] = datetime.date.today().strftime('%Y-%m-%d')
doc['ansible_version'] = ansible_version
doc['plainexamples'] = module_map[module]['examples'] # plain text
doc['metadata'] = module_map[module]['metadata']
if module_map[module]['returndocs']:
try:
doc['returndocs'] = yaml.safe_load(module_map[module]['returndocs'])
except:
print("could not load yaml: %s" % module_map[module]['returndocs'])
raise
else:
doc['returndocs'] = None
text = templates['plugin'].render(doc)
write_data(text, output_dir, outputname, module)
def process_categories(mod_info, categories, templates, output_dir, output_name):
for category in sorted(categories.keys()):
module_map = categories[category]
category_filename = output_name % category
print("*** recording category %s in %s ***" % (category, category_filename))
# start a new category file
category = category.replace("_", " ")
category = category.title()
subcategories = dict((k, v) for k, v in module_map.items() if k != '_modules')
template_data = {'title': category,
'category': module_map,
'subcategories': subcategories,
'module_info': mod_info,
}
text = templates['list_of_CATEGORY_modules'].render(template_data)
write_data(text, output_dir, category_filename)
def process_support_levels(mod_info, templates, output_dir):
supported_by = {'Ansible Core Team': {'slug': 'core_supported',
'modules': [],
'output': 'core_maintained.rst'},
'Ansible Network Team': {'slug': 'network_supported',
'modules': [],
'output': 'network_maintained.rst'},
'Ansible Partners': {'slug': 'partner_supported',
'modules': [],
'output': 'partner_maintained.rst'},
'Ansible Community': {'slug': 'community_supported',
'modules': [],
'output': 'community_maintained.rst'},
}
# Separate the modules by support_level
for module, info in mod_info.items():
if info['metadata']['supported_by'] == 'core':
supported_by['Ansible Core Team']['modules'].append(module)
elif info['metadata']['supported_by'] == 'network':
supported_by['Ansible Network Team']['modules'].append(module)
elif info['metadata']['supported_by'] == 'certified':
supported_by['Ansible Partners']['modules'].append(module)
elif info['metadata']['supported_by'] == 'community':
supported_by['Ansible Community']['modules'].append(module)
else:
raise AnsibleError('Unknown supported_by value: %s' % info['metadata']['supported_by'])
# Render the module lists
for maintainers, data in supported_by.items():
template_data = {'maintainers': maintainers,
'modules': data['modules'],
'slug': data['slug'],
'module_info': mod_info,
}
text = templates['support_list'].render(template_data)
write_data(text, output_dir, data['output'])
def validate_options(options):
''' validate option parser options '''
if not options.module_dir:
sys.exit("--module-dir is required", file=sys.stderr)
if not os.path.exists(options.module_dir):
sys.exit("--module-dir does not exist: %s" % options.module_dir, file=sys.stderr)
if not options.template_dir:
sys.exit("--template-dir must be specified")
def main():
p = generate_parser()
(options, args) = p.parse_args()
validate_options(options)
templates, outputname = jinja2_environment(options.template_dir, options.type)
# Convert passed-in limit_to_modules to None or list of modules.
if options.limit_to_modules is not None:
options.limit_to_modules = [s.lower() for s in options.limit_to_modules.split(",")]
mod_info, categories = get_module_info(options.module_dir, limit_to_modules=options.limit_to_modules,
verbose=options.verbose)
categories['all'] = {'_modules': mod_info.keys()}
# Transform the data
if options.type == 'rst':
for record in mod_info.values():
record['doc']['short_description'] = rst_ify(record['doc']['short_description'])
# Write master category list
category_list_text = templates['category_list'].render(categories=sorted(categories.keys()))
write_data(category_list_text, options.output_dir, 'modules_by_category.rst')
# Render all the individual module pages
process_modules(mod_info, templates, outputname, options.output_dir, options.ansible_version)
# Render all the categories for modules
process_categories(mod_info, categories, templates, options.output_dir, "list_of_%s_modules.rst")
# Render all the categories for modules
process_support_levels(mod_info, templates, options.output_dir)
if __name__ == '__main__':
main()
Reference in a new issue View git blame Copy permalink