plugin_formatter.py: Improve the output when processing docs (#46541)

* Improve the output when processing files

* Update docs/bin/plugin_formatter.py

Co-Authored-By: dagwieers <dag@wieers.com>

* Show progress indicator.

* Don't pp.pformat() huge structures when they aren't used anyway.

This saves ~10 seconds on my machine.

* Only show ASCII spinner if stdout is a TTY.

* Fix: E722 do not use bare 'except'

docs/bin/plugin_formatter.py

@@ -185,11 +185,27 @@ def write_data(text, output_dir, outputname, module=None):
         fname = os.path.join(output_dir, outputname)
         fname = fname.replace(".py", "")
 
-        update_file_if_different(fname, to_bytes(text))
+        try:
+            updated = update_file_if_different(fname, to_bytes(text))
+        except Exception as e:
+            display.display("while rendering %s, an error occured: %s" % (module, e))
+            raise
+        if updated:
+            display.display("rendering: %s" % module)
     else:
         print(text)
 
 
+IS_STDOUT_TTY = sys.stdout.isatty()
+
+
+def show_progress(progress):
+    '''Show a little process indicator.'''
+    if IS_STDOUT_TTY:
+        sys.stdout.write('\r%s\r' % ("-/|\\"[progress % 4]))
+        sys.stdout.flush()
+
+
 def get_plugin_info(module_dir, limit_to=None, verbose=False):
     '''
     Returns information about plugins and the categories that they belong to
@@ -231,6 +247,7 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
         glob.glob("%s/*/*/*/*.py" % module_dir)
     )
 
+    module_index = 0
     for module_path in files:
         # Do not list __init__.py files
         if module_path.endswith('__init__.py'):
@@ -266,6 +283,9 @@ def get_plugin_info(module_dir, limit_to=None, verbose=False):
         # Regular module to process
         #
 
+        module_index += 1
+        show_progress(module_index)
+
         # use ansible core library to parse out doc metadata YAML and plaintext examples
         doc, examples, returndocs, metadata = plugin_docs.get_docstring(module_path, fragment_loader, verbose=verbose)
 
@@ -399,9 +419,10 @@ def too_old(added):
 
 
 def process_plugins(module_map, templates, outputname, output_dir, ansible_version, plugin_type):
-    for module in module_map:
+    for module_index, module in enumerate(module_map):
+
+        show_progress(module_index)
 
-        display.display("rendering: %s" % module)
         fname = module_map[module]['path']
         display.vvvvv(pp.pformat(('process_plugins info: ', module_map[module])))
 
@@ -658,6 +679,8 @@ def main():
     display.verbosity = options.verbosity
     plugin_type = options.plugin_type
 
+    display.display("Evaluating %s files..." % plugin_type)
+
     # prep templating
     templates = jinja2_environment(options.template_dir, options.type, plugin_type)
 
@@ -682,7 +705,9 @@ def main():
 
     categories['all'] = {'_modules': plugin_info.keys()}
 
+    if display.verbosity >= 3:
         display.vvv(pp.pformat(categories))
+    if display.verbosity >= 5:
         display.vvvvv(pp.pformat(plugin_info))
 
     # Transform the data
@@ -690,6 +715,7 @@ def main():
         display.v('Generating rst')
         for key, record in plugin_info.items():
             display.vv(key)
+            if display.verbosity >= 5:
                 display.vvvvv(pp.pformat(('record', record)))
             if record.get('doc', None):
                 short_desc = record['doc']['short_description'].rstrip('.')

lib/ansible/utils/_build_helpers.py

@@ -36,3 +36,6 @@ def update_file_if_different(filename, b_data):
     if b_data_old != b_data:
         with open(filename, 'wb') as f:
             f.write(b_data)
+        return True
+
+    return False