Fixes to the documentation build (#15356)
* Could only have one alias before. Subsequent aliases overrode the previous ones. Now multiple aliases work. * Fix BLACKLISTED_MODULES. Previously, modules were listed in the generated documentation despite being blacklisted * Deprecated modules form extras were showing the (E) tag and not the (D) tag. Reversed that now (Probably not necessary to also show the E tag). * Sort the deprecated modules alphabetically in the Category docs as well as the list of all modules * Optimization: Previously rendered the modules to rst twice once in all group and once in individual categories. Fixed to only render them once. * Add fireball to blacklist and remove async_status (as people need to use that).
This commit is contained in:
Toshio Kuratomi
parent
8d60b298a4
commit
b27c424fa1
2 changed files with 98 additions and 78 deletions
|
|
@ -19,6 +19,7 @@
|
|||
#
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import glob
|
||||
import sys
|
||||
|
|
@ -28,6 +29,8 @@ import optparse
|
|||
import datetime
|
||||
import cgi
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from six import iteritems
|
||||
|
||||
|
|
@ -126,55 +129,45 @@ def write_data(text, options, outputname, module):
|
|||
def list_modules(module_dir, depth=0):
|
||||
''' returns a hash of categories, each category being a hash of module names to file paths '''
|
||||
|
||||
categories = dict(all=dict(),_aliases=dict())
|
||||
if depth <= 3: # limit # of subdirs
|
||||
categories = dict()
|
||||
module_info = dict()
|
||||
aliases = defaultdict(set)
|
||||
|
||||
files = glob.glob("%s/*" % module_dir)
|
||||
for d in files:
|
||||
# * windows powershell modules have documentation stubs in python docstring
|
||||
# format (they are not executed) so skip the ps1 format files
|
||||
# * One glob level for every module level that we're going to traverse
|
||||
files = glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir)
|
||||
|
||||
category = os.path.splitext(os.path.basename(d))[0]
|
||||
if os.path.isdir(d):
|
||||
for module_path in files:
|
||||
if module_path.endswith('__init__.py'):
|
||||
continue
|
||||
category = categories
|
||||
mod_path_only = os.path.dirname(module_path[len(module_dir) + 1:])
|
||||
# Start at the second directory because we don't want the "vendor"
|
||||
# directories (core, extras)
|
||||
for new_cat in mod_path_only.split('/')[1:]:
|
||||
if new_cat not in category:
|
||||
category[new_cat] = dict()
|
||||
category = category[new_cat]
|
||||
|
||||
res = list_modules(d, depth + 1)
|
||||
for key in list(res.keys()):
|
||||
if key in categories:
|
||||
categories[key] = merge_hash(categories[key], res[key])
|
||||
res.pop(key, None)
|
||||
module = os.path.splitext(os.path.basename(module_path))[0]
|
||||
if module in module_docs.BLACKLIST_MODULES:
|
||||
# Do not list blacklisted modules
|
||||
continue
|
||||
if module.startswith("_") and os.path.islink(module_path):
|
||||
source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
|
||||
module = module.replace("_","",1)
|
||||
aliases[source].add(module)
|
||||
continue
|
||||
|
||||
if depth < 2:
|
||||
categories.update(res)
|
||||
else:
|
||||
category = module_dir.split("/")[-1]
|
||||
if not category in categories:
|
||||
categories[category] = res
|
||||
else:
|
||||
categories[category].update(res)
|
||||
else:
|
||||
module = category
|
||||
category = os.path.basename(module_dir)
|
||||
if not d.endswith(".py") or d.endswith('__init__.py'):
|
||||
# windows powershell modules have documentation stubs in python docstring
|
||||
# format (they are not executed) so skip the ps1 format files
|
||||
continue
|
||||
elif module.startswith("_") and os.path.islink(d):
|
||||
source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0]
|
||||
module = module.replace("_","",1)
|
||||
if not d in categories['_aliases']:
|
||||
categories['_aliases'][source] = [module]
|
||||
else:
|
||||
categories['_aliases'][source].update(module)
|
||||
continue
|
||||
category[module] = module_path
|
||||
module_info[module] = module_path
|
||||
|
||||
if not category in categories:
|
||||
categories[category] = {}
|
||||
categories[category][module] = d
|
||||
categories['all'][module] = d
|
||||
|
||||
# keep module tests out of becomeing module docs
|
||||
# keep module tests out of becoming module docs
|
||||
if 'test' in categories:
|
||||
del categories['test']
|
||||
|
||||
return categories
|
||||
return module_info, categories, aliases
|
||||
|
||||
#####################################################################################
|
||||
|
||||
|
|
@ -258,11 +251,8 @@ def process_module(module, options, env, template, outputname, module_map, alias
|
|||
|
||||
# crash if module is missing documentation and not explicitly hidden from docs index
|
||||
if doc is None:
|
||||
if module in module_docs.BLACKLIST_MODULES:
|
||||
return "SKIPPED"
|
||||
else:
|
||||
sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
|
||||
sys.exit(1)
|
||||
sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
|
||||
sys.exit(1)
|
||||
|
||||
if deprecated and 'deprecated' not in doc:
|
||||
sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
|
||||
|
|
@ -334,21 +324,32 @@ def process_module(module, options, env, template, outputname, module_map, alias
|
|||
|
||||
def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases):
|
||||
modstring = module
|
||||
modname = module
|
||||
if modstring.startswith('_'):
|
||||
modstring = module[1:]
|
||||
modname = modstring
|
||||
if module in deprecated:
|
||||
modstring = modstring + DEPRECATED
|
||||
modname = "_" + module
|
||||
elif module not in core:
|
||||
modstring = modstring + NOTCORE
|
||||
|
||||
result = process_module(modname, options, env, template, outputname, module_map, aliases)
|
||||
|
||||
if result != "SKIPPED":
|
||||
category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(result)), to_bytes(module)))
|
||||
category_file.write(" %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
|
||||
|
||||
def process_category(category, categories, options, env, template, outputname):
|
||||
|
||||
### FIXME:
|
||||
# We no longer conceptually deal with a mapping of category names to
|
||||
# modules to file paths. Instead we want several different records:
|
||||
# (1) Mapping of module names to file paths (what's presently used
|
||||
# as categories['all']
|
||||
# (2) Mapping of category names to lists of module names (what you'd
|
||||
# presently get from categories[category_name][subcategory_name].keys()
|
||||
# (3) aliases (what's presently in categories['_aliases']
|
||||
#
|
||||
# list_modules() now returns those. Need to refactor this function and
|
||||
# main to work with them.
|
||||
|
||||
module_map = categories[category]
|
||||
module_info = categories['all']
|
||||
|
||||
aliases = {}
|
||||
if '_aliases' in categories:
|
||||
|
|
@ -369,21 +370,21 @@ def process_category(category, categories, options, env, template, outputname):
|
|||
for module in module_map.keys():
|
||||
|
||||
if isinstance(module_map[module], dict):
|
||||
for mod in module_map[module].keys():
|
||||
for mod in (m for m in module_map[module].keys() if m in module_info):
|
||||
if mod.startswith("_"):
|
||||
mod = mod.replace("_","",1)
|
||||
deprecated.append(mod)
|
||||
elif '/core/' in module_map[module][mod]:
|
||||
elif '/core/' in module_info[mod][0]:
|
||||
core.append(mod)
|
||||
else:
|
||||
if module not in module_info:
|
||||
continue
|
||||
if module.startswith("_"):
|
||||
module = module.replace("_","",1)
|
||||
deprecated.append(module)
|
||||
elif '/core/' in module_map[module]:
|
||||
elif '/core/' in module_info[module][0]:
|
||||
core.append(module)
|
||||
modules.append(module)
|
||||
|
||||
modules.sort()
|
||||
modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
|
||||
|
||||
category_header = "%s Modules" % (category.title())
|
||||
underscores = "`" * len(category_header)
|
||||
|
|
@ -401,7 +402,7 @@ def process_category(category, categories, options, env, template, outputname):
|
|||
sections.append(module)
|
||||
continue
|
||||
else:
|
||||
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases)
|
||||
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)
|
||||
|
||||
sections.sort()
|
||||
for section in sections:
|
||||
|
|
@ -409,10 +410,10 @@ def process_category(category, categories, options, env, template, outputname):
|
|||
category_file.write(".. toctree:: :maxdepth: 1\n\n")
|
||||
|
||||
section_modules = module_map[section].keys()
|
||||
section_modules.sort()
|
||||
section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
|
||||
#for module in module_map[section]:
|
||||
for module in section_modules:
|
||||
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases)
|
||||
for module in (m for m in section_modules if m in module_info):
|
||||
print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)
|
||||
|
||||
category_file.write("""\n\n
|
||||
.. note::
|
||||
|
|
@ -450,25 +451,42 @@ def main():
|
|||
|
||||
env, template, outputname = jinja2_environment(options.template_dir, options.type)
|
||||
|
||||
categories = list_modules(options.module_dir)
|
||||
category_names = list(categories.keys())
|
||||
mod_info, categories, aliases = list_modules(options.module_dir)
|
||||
categories['all'] = mod_info
|
||||
categories['_aliases'] = aliases
|
||||
category_names = [c for c in categories.keys() if not c.startswith('_')]
|
||||
category_names.sort()
|
||||
|
||||
# Write master category list
|
||||
category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
|
||||
category_list_file = open(category_list_path, "w")
|
||||
category_list_file.write("Module Index\n")
|
||||
category_list_file.write("============\n")
|
||||
category_list_file.write("\n\n")
|
||||
category_list_file.write(".. toctree::\n")
|
||||
category_list_file.write(" :maxdepth: 1\n\n")
|
||||
with open(category_list_path, "w") as category_list_file:
|
||||
category_list_file.write("Module Index\n")
|
||||
category_list_file.write("============\n")
|
||||
category_list_file.write("\n\n")
|
||||
category_list_file.write(".. toctree::\n")
|
||||
category_list_file.write(" :maxdepth: 1\n\n")
|
||||
|
||||
for category in category_names:
|
||||
category_list_file.write(" list_of_%s_modules\n" % category)
|
||||
|
||||
#
|
||||
# Import all the docs into memory
|
||||
#
|
||||
module_map = mod_info.copy()
|
||||
skipped_modules = set()
|
||||
|
||||
for modname in module_map:
|
||||
result = process_module(modname, options, env, template, outputname, module_map, aliases)
|
||||
if result == 'SKIPPED':
|
||||
del categories['all'][modname]
|
||||
else:
|
||||
categories['all'][modname] = (categories['all'][modname], result)
|
||||
|
||||
#
|
||||
# Render all the docs to rst via category pages
|
||||
#
|
||||
for category in category_names:
|
||||
if category.startswith("_"):
|
||||
continue
|
||||
category_list_file.write(" list_of_%s_modules\n" % category)
|
||||
process_category(category, categories, options, env, template, outputname)
|
||||
|
||||
category_list_file.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -37,9 +37,11 @@ except ImportError:
|
|||
display = Display()
|
||||
|
||||
# modules that are ok that they do not have documentation strings
|
||||
BLACKLIST_MODULES = [
|
||||
'async_wrapper', 'accelerate', 'async_status'
|
||||
]
|
||||
BLACKLIST_MODULES = frozenset((
|
||||
'async_wrapper',
|
||||
'accelerate',
|
||||
'fireball',
|
||||
))
|
||||
|
||||
def get_docstring(filename, verbose=False):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Reference in a new issue