SCons: Add an option to detect C++ modules recursively

This adds `custom_modules_recursive` which allows to detect and collect
all nested C++ modules which may reside in any directory specified by
`custom_modules` option.

The detection logic is made to be more strict because `SCSub` may be
used for organizing hierarchical builds within a module itself, so the
existence of `register_types.h` and `config.py` is checked as well
(these are all required for a C++ module to be compiled by Godot).

For performance reasons, built-in modules are not checked recursively,
and there's no benefit of doing so in the first place.

It's now possible to specify a directory path pointing to a *single*
module, as it may contain nested modules which are detected recursively.

SConstruct

@@ -123,6 +123,7 @@ opts.Add(BoolVariable("deprecated", "Enable deprecated features", True))
 opts.Add(BoolVariable("minizip", "Enable ZIP archive support using minizip", True))
 opts.Add(BoolVariable("xaudio2", "Enable the XAudio2 audio driver", False))
 opts.Add("custom_modules", "A list of comma-separated directory paths containing custom modules to build.", "")
+opts.Add(BoolVariable("custom_modules_recursive", "Detect custom modules recursively for each specified path.", True))
 
 # Advanced options
 opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
@@ -199,8 +200,14 @@ if env_base["custom_modules"]:
             Exit(255)
 
 for path in module_search_paths:
+    if path == "modules":
+        # Built-in modules don't have nested modules,
+        # so save the time it takes to parse directories.
+        modules = methods.detect_modules(path, recursive=False)
+    else:  # External.
+        modules = methods.detect_modules(path, env_base["custom_modules_recursive"])
     # Note: custom modules can override built-in ones.
-    modules_detected.update(methods.detect_modules(path))
+    modules_detected.update(modules)
     include_path = os.path.dirname(path)
     if include_path:
         env_base.Prepend(CPPPATH=[include_path])

methods.py

@@ -145,34 +145,88 @@ def parse_cg_file(fname, uniforms, sizes, conditionals):
     fs.close()
 
 
-def detect_modules(at_path):
+def detect_modules(search_path, recursive=False):
-    module_list = OrderedDict()  # name : path
+    """Detects and collects a list of C++ modules at specified path
 
-    modules_glob = os.path.join(at_path, "*")
+    `search_path` - a directory path containing modules. The path may point to
-    files = glob.glob(modules_glob)
+    a single module, which may have other nested modules. A module must have
-    files.sort()  # so register_module_types does not change that often, and also plugins are registered in alphabetic order
+    "register_types.h", "SCsub", "config.py" files created to be detected.
 
-    for x in files:
+    `recursive` - if `True`, then all subdirectories are searched for modules as
-        if not is_module(x):
+    specified by the `search_path`, otherwise collects all modules under the
-            continue
+    `search_path` directory. If the `search_path` is a module, it is collected
-        name = os.path.basename(x)
+    in all cases.
-        path = x.replace("\\", "/")  # win32
-        module_list[name] = path
 
-    return module_list
+    Returns an `OrderedDict` with module names as keys, and directory paths as
+    values. If a path is relative, then it is a built-in module. If a path is
+    absolute, then it is a custom module collected outside of the engine source.
+    """
+    modules = OrderedDict()
+
+    def add_module(path):
+        module_name = os.path.basename(path)
+        module_path = path.replace("\\", "/")  # win32
+        modules[module_name] = module_path
+
+    def is_engine(path):
+        # Prevent recursively detecting modules in self and other
+        # Godot sources when using `custom_modules` build option.
+        version_path = os.path.join(path, "version.py")
+        if os.path.exists(version_path):
+            with open(version_path) as f:
+                version = {}
+                exec(f.read(), version)
+                if version.get("short_name") == "godot":
+                    return True
+        return False
+
+    def get_files(path):
+        files = glob.glob(os.path.join(path, "*"))
+        # Sort so that `register_module_types` does not change that often,
+        # and plugins are registered in alphabetic order as well.
+        files.sort()
+        return files
+
+    if not recursive:
+        if is_module(search_path):
+            add_module(search_path)
+        for path in get_files(search_path):
+            if is_engine(path):
+                continue
+            if is_module(path):
+                add_module(path)
+    else:
+        to_search = [search_path]
+        while to_search:
+            path = to_search.pop()
+            if is_module(path):
+                add_module(path)
+            for child in get_files(path):
+                if not os.path.isdir(child):
+                    continue
+                if is_engine(child):
+                    continue
+                to_search.insert(0, child)
+    return modules
 
 
 def is_module(path):
-    return os.path.isdir(path) and os.path.exists(os.path.join(path, "SCsub"))
+    if not os.path.isdir(path):
+        return False
+    must_exist = ["register_types.h", "SCsub", "config.py"]
+    for f in must_exist:
+        if not os.path.exists(os.path.join(path, f)):
+            return False
+    return True
 
 
-def write_modules(module_list):
+def write_modules(modules):
     includes_cpp = ""
     preregister_cpp = ""
     register_cpp = ""
     unregister_cpp = ""
 
-    for name, path in module_list.items():
+    for name, path in modules.items():
         try:
             with open(os.path.join(path, "register_types.h")):
                 includes_cpp += '#include "' + path + '/register_types.h"\n'
@@ -230,8 +284,6 @@ def convert_custom_modules_path(path):
         raise ValueError(err_msg % "point to an existing directory.")
     if path == os.path.realpath("modules"):
         raise ValueError(err_msg % "be a directory other than built-in `modules` directory.")
-    if is_module(path):
-        raise ValueError(err_msg % "point to a directory with modules, not a single module.")
     return path