From b27c424fa1ae0af8ea63e909504774d2b07cf916 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Mon, 11 Apr 2016 17:11:55 -0700 Subject: [PATCH] Fixes to the documentation build (#15356) * Could only have one alias before. Subsequent aliases overrode the previous ones. Now multiple aliases work. * Fix BLACKLISTED_MODULES. Previously, modules were listed in the generated documentation despite being blacklisted * Deprecated modules form extras were showing the (E) tag and not the (D) tag. Reversed that now (Probably not necessary to also show the E tag). * Sort the deprecated modules alphabetically in the Category docs as well as the list of all modules * Optimization: Previously rendered the modules to rst twice once in all group and once in individual categories. Fixed to only render them once. * Add fireball to blacklist and remove async_status (as people need to use that). --- hacking/module_formatter.py | 168 +++++++++++++++++-------------- lib/ansible/utils/module_docs.py | 8 +- 2 files changed, 98 insertions(+), 78 deletions(-) diff --git a/hacking/module_formatter.py b/hacking/module_formatter.py index 011bfd53c4a..07a9c540c16 100755 --- a/hacking/module_formatter.py +++ b/hacking/module_formatter.py @@ -19,6 +19,7 @@ # from __future__ import print_function + import os import glob import sys @@ -28,6 +29,8 @@ import optparse import datetime import cgi import warnings +from collections import defaultdict + from jinja2 import Environment, FileSystemLoader from six import iteritems @@ -126,55 +129,45 @@ def write_data(text, options, outputname, module): def list_modules(module_dir, depth=0): ''' returns a hash of categories, each category being a hash of module names to file paths ''' - categories = dict(all=dict(),_aliases=dict()) - if depth <= 3: # limit # of subdirs + categories = dict() + module_info = dict() + aliases = defaultdict(set) - files = glob.glob("%s/*" % module_dir) - for d in files: + # * windows powershell modules have documentation stubs in python docstring + # format (they are not executed) so skip the ps1 format files + # * One glob level for every module level that we're going to traverse + files = glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir) - category = os.path.splitext(os.path.basename(d))[0] - if os.path.isdir(d): + for module_path in files: + if module_path.endswith('__init__.py'): + continue + category = categories + mod_path_only = os.path.dirname(module_path[len(module_dir) + 1:]) + # Start at the second directory because we don't want the "vendor" + # directories (core, extras) + for new_cat in mod_path_only.split('/')[1:]: + if new_cat not in category: + category[new_cat] = dict() + category = category[new_cat] - res = list_modules(d, depth + 1) - for key in list(res.keys()): - if key in categories: - categories[key] = merge_hash(categories[key], res[key]) - res.pop(key, None) + module = os.path.splitext(os.path.basename(module_path))[0] + if module in module_docs.BLACKLIST_MODULES: + # Do not list blacklisted modules + continue + if module.startswith("_") and os.path.islink(module_path): + source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0] + module = module.replace("_","",1) + aliases[source].add(module) + continue - if depth < 2: - categories.update(res) - else: - category = module_dir.split("/")[-1] - if not category in categories: - categories[category] = res - else: - categories[category].update(res) - else: - module = category - category = os.path.basename(module_dir) - if not d.endswith(".py") or d.endswith('__init__.py'): - # windows powershell modules have documentation stubs in python docstring - # format (they are not executed) so skip the ps1 format files - continue - elif module.startswith("_") and os.path.islink(d): - source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0] - module = module.replace("_","",1) - if not d in categories['_aliases']: - categories['_aliases'][source] = [module] - else: - categories['_aliases'][source].update(module) - continue + category[module] = module_path + module_info[module] = module_path - if not category in categories: - categories[category] = {} - categories[category][module] = d - categories['all'][module] = d - - # keep module tests out of becomeing module docs + # keep module tests out of becoming module docs if 'test' in categories: del categories['test'] - return categories + return module_info, categories, aliases ##################################################################################### @@ -258,11 +251,8 @@ def process_module(module, options, env, template, outputname, module_map, alias # crash if module is missing documentation and not explicitly hidden from docs index if doc is None: - if module in module_docs.BLACKLIST_MODULES: - return "SKIPPED" - else: - sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) - sys.exit(1) + sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module)) + sys.exit(1) if deprecated and 'deprecated' not in doc: sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module)) @@ -334,21 +324,32 @@ def process_module(module, options, env, template, outputname, module_map, alias def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases): modstring = module - modname = module + if modstring.startswith('_'): + modstring = module[1:] + modname = modstring if module in deprecated: modstring = modstring + DEPRECATED - modname = "_" + module elif module not in core: modstring = modstring + NOTCORE - result = process_module(modname, options, env, template, outputname, module_map, aliases) - - if result != "SKIPPED": - category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(result)), to_bytes(module))) + category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname))) def process_category(category, categories, options, env, template, outputname): + ### FIXME: + # We no longer conceptually deal with a mapping of category names to + # modules to file paths. Instead we want several different records: + # (1) Mapping of module names to file paths (what's presently used + # as categories['all'] + # (2) Mapping of category names to lists of module names (what you'd + # presently get from categories[category_name][subcategory_name].keys() + # (3) aliases (what's presently in categories['_aliases'] + # + # list_modules() now returns those. Need to refactor this function and + # main to work with them. + module_map = categories[category] + module_info = categories['all'] aliases = {} if '_aliases' in categories: @@ -369,21 +370,21 @@ def process_category(category, categories, options, env, template, outputname): for module in module_map.keys(): if isinstance(module_map[module], dict): - for mod in module_map[module].keys(): + for mod in (m for m in module_map[module].keys() if m in module_info): if mod.startswith("_"): - mod = mod.replace("_","",1) deprecated.append(mod) - elif '/core/' in module_map[module][mod]: + elif '/core/' in module_info[mod][0]: core.append(mod) else: + if module not in module_info: + continue if module.startswith("_"): - module = module.replace("_","",1) deprecated.append(module) - elif '/core/' in module_map[module]: + elif '/core/' in module_info[module][0]: core.append(module) modules.append(module) - modules.sort() + modules.sort(key=lambda k: k[1:] if k.startswith('_') else k) category_header = "%s Modules" % (category.title()) underscores = "`" * len(category_header) @@ -401,7 +402,7 @@ def process_category(category, categories, options, env, template, outputname): sections.append(module) continue else: - print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases) + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases) sections.sort() for section in sections: @@ -409,10 +410,10 @@ def process_category(category, categories, options, env, template, outputname): category_file.write(".. toctree:: :maxdepth: 1\n\n") section_modules = module_map[section].keys() - section_modules.sort() + section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k) #for module in module_map[section]: - for module in section_modules: - print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases) + for module in (m for m in section_modules if m in module_info): + print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases) category_file.write("""\n\n .. note:: @@ -450,25 +451,42 @@ def main(): env, template, outputname = jinja2_environment(options.template_dir, options.type) - categories = list_modules(options.module_dir) - category_names = list(categories.keys()) + mod_info, categories, aliases = list_modules(options.module_dir) + categories['all'] = mod_info + categories['_aliases'] = aliases + category_names = [c for c in categories.keys() if not c.startswith('_')] category_names.sort() + # Write master category list category_list_path = os.path.join(options.output_dir, "modules_by_category.rst") - category_list_file = open(category_list_path, "w") - category_list_file.write("Module Index\n") - category_list_file.write("============\n") - category_list_file.write("\n\n") - category_list_file.write(".. toctree::\n") - category_list_file.write(" :maxdepth: 1\n\n") + with open(category_list_path, "w") as category_list_file: + category_list_file.write("Module Index\n") + category_list_file.write("============\n") + category_list_file.write("\n\n") + category_list_file.write(".. toctree::\n") + category_list_file.write(" :maxdepth: 1\n\n") + for category in category_names: + category_list_file.write(" list_of_%s_modules\n" % category) + + # + # Import all the docs into memory + # + module_map = mod_info.copy() + skipped_modules = set() + + for modname in module_map: + result = process_module(modname, options, env, template, outputname, module_map, aliases) + if result == 'SKIPPED': + del categories['all'][modname] + else: + categories['all'][modname] = (categories['all'][modname], result) + + # + # Render all the docs to rst via category pages + # for category in category_names: - if category.startswith("_"): - continue - category_list_file.write(" list_of_%s_modules\n" % category) process_category(category, categories, options, env, template, outputname) - category_list_file.close() - if __name__ == '__main__': main() diff --git a/lib/ansible/utils/module_docs.py b/lib/ansible/utils/module_docs.py index e647da00143..e98df1ba069 100755 --- a/lib/ansible/utils/module_docs.py +++ b/lib/ansible/utils/module_docs.py @@ -37,9 +37,11 @@ except ImportError: display = Display() # modules that are ok that they do not have documentation strings -BLACKLIST_MODULES = [ - 'async_wrapper', 'accelerate', 'async_status' -] +BLACKLIST_MODULES = frozenset(( + 'async_wrapper', + 'accelerate', + 'fireball', +)) def get_docstring(filename, verbose=False): """