diff --git a/lib/ansible/module_utils/csharp/Ansible.Basic.cs b/lib/ansible/module_utils/csharp/Ansible.Basic.cs
index ef92c605edb..90d788bfcf5 100644
--- a/lib/ansible/module_utils/csharp/Ansible.Basic.cs
+++ b/lib/ansible/module_utils/csharp/Ansible.Basic.cs
@@ -38,6 +38,8 @@ namespace Ansible.Basic
         public delegate void WriteLineHandler(string line);
         public static WriteLineHandler WriteLine = new WriteLineHandler(WriteLineModule);
 
+        public static bool _DebugArgSpec = false;
+
         private static List<string> BOOLEANS_TRUE = new List<string>() { "y", "yes", "on", "1", "true", "t", "1.0" };
         private static List<string> BOOLEANS_FALSE = new List<string>() { "n", "no", "off", "0", "false", "f", "0.0" };
 
@@ -193,12 +195,30 @@ namespace Ansible.Basic
             try
             {
                 ValidateArgumentSpec(argumentSpec);
+
+                // Used by ansible-test to retrieve the module argument spec, not designed for public use.
+                if (_DebugArgSpec)
+                {
+                    // Cannot call exit here because it will be caught with the catch (Exception e) below. Instead
+                    // just throw a new exception with a specific message and the exception block will handle it.
+                    ScriptBlock.Create("Set-Variable -Name ansibleTestArgSpec -Value $args[0] -Scope Global"
+                        ).Invoke(argumentSpec);
+                    throw new Exception("ansible-test validate-modules check");
+                }
+
+                // Now make sure all the metadata keys are set to their defaults, this must be done after we've
+                // potentially output the arg spec for ansible-test.
+                SetArgumentSpecDefaults(argumentSpec);
+
                 Params = GetParams(args);
                 aliases = GetAliases(argumentSpec, Params);
                 SetNoLogValues(argumentSpec, Params);
             }
             catch (Exception e)
             {
+                if (e.Message == "ansible-test validate-modules check")
+                    Exit(0);
+
                 Dictionary<string, object> result = new Dictionary<string, object>
                 {
                     { "failed", true },
@@ -657,8 +677,10 @@ namespace Ansible.Basic
             // Outside of the spec iterator, change the values that were casted above
             foreach (KeyValuePair<string, object> changedValue in changedValues)
                 argumentSpec[changedValue.Key] = changedValue.Value;
+        }
 
-            // Now make sure all the metadata keys are set to their defaults
+        private void SetArgumentSpecDefaults(IDictionary argumentSpec)
+        {
             foreach (KeyValuePair<string, List<object>> metadataEntry in specDefaults)
             {
                 List<object> defaults = metadataEntry.Value;
@@ -669,6 +691,22 @@ namespace Ansible.Basic
                 if (!argumentSpec.Contains(metadataEntry.Key))
                     argumentSpec[metadataEntry.Key] = defaultValue;
             }
+
+            // Recursively set the defaults for any inner options.
+            foreach (DictionaryEntry entry in argumentSpec)
+            {
+                if (entry.Value == null || entry.Key.ToString() != "options")
+                    continue;
+
+                IDictionary optionsSpec = (IDictionary)entry.Value;
+                foreach (DictionaryEntry optionEntry in optionsSpec)
+                {
+                    optionsContext.Add((string)optionEntry.Key);
+                    IDictionary optionMeta = (IDictionary)optionEntry.Value;
+                    SetArgumentSpecDefaults(optionMeta);
+                    optionsContext.RemoveAt(optionsContext.Count - 1);
+                }
+            }
         }
 
         private Dictionary<string, string> GetAliases(IDictionary argumentSpec, IDictionary parameters)
diff --git a/shippable.yml b/shippable.yml
index 18b38cd61b3..96561961f17 100644
--- a/shippable.yml
+++ b/shippable.yml
@@ -110,6 +110,9 @@ matrix:
     - env: T=fallaxy/2.7/1
     - env: T=fallaxy/3.6/1
 
+    - env: T=generic/2.7/1
+    - env: T=generic/3.6/1
+
     - env: T=i/osx/10.11
     - env: T=i/rhel/7.8
     - env: T=i/rhel/8.2
diff --git a/test/integration/targets/ansible-test-docker/aliases b/test/integration/targets/ansible-test-docker/aliases
new file mode 100644
index 00000000000..d1284cf706f
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/aliases
@@ -0,0 +1 @@
+shippable/generic/group1  # Runs in the default test container so access to tools like pwsh
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/doc_fragments/ps_util.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/doc_fragments/ps_util.py
new file mode 100644
index 00000000000..e69844b349d
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/doc_fragments/ps_util.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+class ModuleDocFragment:
+
+    DOCUMENTATION = r'''
+options:
+  option1:
+    description:
+    - Test description
+    required: yes
+    aliases:
+    - alias1
+    type: str
+'''
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/PSUtil.psm1 b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/PSUtil.psm1
new file mode 100644
index 00000000000..d37e681a8ac
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/PSUtil.psm1
@@ -0,0 +1,16 @@
+# Copyright (c) 2020 Ansible Project
+# # Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause)
+
+Function Get-PSUtilSpec {
+    <#
+    .SYNOPSIS
+    Shared util spec test
+    #>
+    @{
+        options = @{
+          option1 = @{ type = 'str'; required = $true; aliases = 'alias1' }
+        }
+    }
+}
+
+Export-ModuleMember -Function Get-PSUtilSpec
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/my_util.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/my_util.py
new file mode 100644
index 00000000000..b9c531cf8aa
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/module_utils/my_util.py
@@ -0,0 +1,6 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+def hello(name):
+    return 'Hello %s' % name
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py
new file mode 100644
index 00000000000..c8a0cf75a03
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: hello
+short_description: Hello test module
+description: Hello test module.
+options:
+  name:
+    description: Name to say hello to.
+    type: str
+author:
+  - Ansible Core Team
+'''
+
+EXAMPLES = '''
+- minimal:
+'''
+
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+from ..module_utils.my_util import hello
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            name=dict(type='str'),
+        ),
+    )
+
+    module.exit_json(**say_hello(module.params['name']))
+
+
+def say_hello(name):
+    return dict(
+        message=hello(name),
+    )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.ps1 b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.ps1
new file mode 100644
index 00000000000..7697725648b
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.ps1
@@ -0,0 +1,18 @@
+#!powershell
+
+# Copyright (c) 2020 Ansible Project
+# # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+#AnsibleRequires -CSharpUtil Ansible.Basic
+#AnsibleRequires -PowerShell ..module_utils.PSUtil
+
+$spec = @{
+    options = @{
+      my_opt = @{ type = "str"; required = $true }
+    }
+}
+$util_spec = Get-PSUtilSpec
+$spec.options += $util_spec.options
+
+$module = [Ansible.Basic.AnsibleModule]::Create($args, $spec)
+$module.ExitJson()
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.py
new file mode 100644
index 00000000000..ed49f4ea815
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/win_util_args.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2020 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['stableinterface'],
+                    'supported_by': 'core'}
+
+DOCUMENTATION = r'''
+---
+module: win_util_args
+short_description: Short description
+description:
+- Some test description for the module
+options:
+  my_opt:
+    description:
+    - Test description
+    required: yes
+    type: str
+extends_documentation_fragment:
+- ns.col.ps_util
+
+author:
+- Ansible Test (@ansible)
+'''
+
+EXAMPLES = r'''
+- win_util_args:
+    option1: test
+    my_opt: test
+'''
+
+RETURN = r'''
+#
+'''
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/tasks/main.yml b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/tasks/main.yml
new file mode 100644
index 00000000000..c45c199cf3f
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/tasks/main.yml
@@ -0,0 +1,7 @@
+- hello:
+    name: Ansibull
+  register: hello
+
+- assert:
+    that:
+      - hello.message == 'Hello Ansibull'
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py
new file mode 100644
index 00000000000..7df87103767
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py
@@ -0,0 +1,8 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from .....plugins.module_utils.my_util import hello
+
+
+def test_hello():
+    assert hello('Ansibull') == 'Hello Ansibull'
diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py
new file mode 100644
index 00000000000..95ee0574fac
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py
@@ -0,0 +1,8 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from .....plugins.modules.hello import say_hello
+
+
+def test_say_hello():
+    assert say_hello('Ansibull') == dict(message='Hello Ansibull')
diff --git a/test/integration/targets/ansible-test-docker/collection-tests/docker.sh b/test/integration/targets/ansible-test-docker/collection-tests/docker.sh
new file mode 100755
index 00000000000..e0e342909e5
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/collection-tests/docker.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+# common args for all tests
+# because we are running in shippable/generic/ we are already in the default docker container
+common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
+
+# prime the venv to work around issue with PyYAML detection in ansible-test
+ansible-test sanity "${common[@]}" --test ignores
+
+# tests
+ansible-test sanity "${common[@]}"
+ansible-test units "${common[@]}"
+ansible-test integration "${common[@]}"
diff --git a/test/integration/targets/ansible-test-docker/runme.sh b/test/integration/targets/ansible-test-docker/runme.sh
new file mode 100755
index 00000000000..7c956b4f158
--- /dev/null
+++ b/test/integration/targets/ansible-test-docker/runme.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# tests must be executed outside of the ansible source tree
+# otherwise ansible-test will test the ansible source instead of the test collection
+# the temporary directory provided by ansible-test resides within the ansible source tree
+tmp_dir=$(mktemp -d)
+
+trap 'rm -rf "${tmp_dir}"' EXIT
+
+export TEST_DIR
+export WORK_DIR
+
+TEST_DIR="$PWD"
+
+for test in collection-tests/*.sh; do
+  WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
+  mkdir "${WORK_DIR}"
+  echo "**********************************************************************"
+  echo "TEST: ${test}: STARTING"
+  "${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
+  echo "TEST: ${test}: PASSED"
+done
diff --git a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/module_args.py b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/module_args.py
index 953b40091db..a2005c5746a 100644
--- a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/module_args.py
+++ b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/module_args.py
@@ -89,9 +89,9 @@ def setup_env(filename):
                 del sys.modules[k]
 
 
-def get_ps_argument_spec(filename):
-    # This uses a very small skeleton of Ansible.Basic.AnsibleModule to return the argspec defined by the module. This
-    # is pretty rudimentary and will probably require something better going forward.
+def get_ps_argument_spec(filename, collection):
+    fqc_name = get_module_name_from_filename(filename, collection)
+
     pwsh = find_executable('pwsh')
     if not pwsh:
         raise FileNotFoundError('Required program for PowerShell arg spec inspection "pwsh" not found.')
@@ -102,11 +102,15 @@ def get_ps_argument_spec(filename):
         b_module_data = module_fd.read()
 
     ps_dep_finder = PSModuleDepFinder()
-    ps_dep_finder.scan_module(b_module_data)
+    ps_dep_finder.scan_module(b_module_data, fqn=fqc_name)
+
+    # For ps_argspec.ps1 to compile Ansible.Basic it also needs the AddType module_util.
+    ps_dep_finder._add_module((b"Ansible.ModuleUtils.AddType", ".psm1", None), wrapper=False)
 
     util_manifest = json.dumps({
         'module_path': to_text(module_path, errors='surrogiate_or_strict'),
-        'ps_utils': dict([(name, info['path']) for name, info in ps_dep_finder.ps_modules.items()])
+        'ansible_basic': ps_dep_finder.cs_utils_module["Ansible.Basic"]['path'],
+        'ps_utils': dict([(name, info['path']) for name, info in ps_dep_finder.ps_modules.items()]),
     })
 
     script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ps_argspec.ps1')
@@ -115,7 +119,7 @@ def get_ps_argument_spec(filename):
     stdout, stderr = proc.communicate()
 
     if proc.returncode != 0:
-        raise AnsibleModuleImportError(stderr.decode('utf-8'))
+        raise AnsibleModuleImportError("STDOUT:\n%s\nSTDERR:\n%s" % (stdout.decode('utf-8'), stderr.decode('utf-8')))
 
     kwargs = json.loads(stdout)
 
@@ -163,4 +167,4 @@ def get_argument_spec(filename, collection):
     if filename.endswith('.py'):
         return get_py_argument_spec(filename, collection)
     else:
-        return get_ps_argument_spec(filename)
+        return get_ps_argument_spec(filename, collection)
diff --git a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/ps_argspec.ps1 b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/ps_argspec.ps1
index 8a67df63f7d..bf3b1f0370d 100755
--- a/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/ps_argspec.ps1
+++ b/test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/ps_argspec.ps1
@@ -63,30 +63,6 @@ if (-not (Test-Path -LiteralPath $module_path -PathType Leaf)) {
     exit 1
 }
 
-$dummy_ansible_basic = @'
-using System;
-using System.Collections;
-using System.Management.Automation;
-
-namespace Ansible.Basic
-{
-    public class AnsibleModule
-    {
-        public AnsibleModule(string[] args, IDictionary argumentSpec)
-        {
-            ScriptBlock.Create("Set-Variable -Name arg_spec -Value $args[0] -Scope Global; exit 0"
-                ).Invoke(new Object[] { argumentSpec });
-        }
-
-        public static AnsibleModule Create(string[] args, IDictionary argumentSpec)
-        {
-            return new AnsibleModule(args, argumentSpec);
-        }
-    }
-}
-'@
-Add-Type -TypeDefinition $dummy_ansible_basic
-
 $module_code = Get-Content -LiteralPath $module_path -Raw
 
 $powershell = [PowerShell]::Create()
@@ -111,9 +87,15 @@ if ($manifest.Contains('ps_utils')) {
         }) > $null
         $powershell.AddCommand('Import-Module').AddParameter('WarningAction', 'SilentlyContinue') > $null
         $powershell.AddCommand('Out-Null').AddStatement() > $null
+
+        # Also import it into the current runspace in case ps_argspec.ps1 needs to use it.
+        $null = New-Module -Name $util_name -ScriptBlock $util_sb | Import-Module -WarningAction SilentlyContinue
     }
 }
 
+Add-CSharpType -References @(Get-Content -LiteralPath $manifest.ansible_basic -Raw)
+[Ansible.Basic.AnsibleModule]::_DebugArgSpec = $true
+
 $powershell.AddScript($module_code) > $null
 $powershell.Invoke() > $null
 
@@ -122,7 +104,7 @@ if ($powershell.HadErrors) {
     exit 1
 }
 
-$arg_spec = $powershell.Runspace.SessionStateProxy.GetVariable('arg_spec')
+$arg_spec = $powershell.Runspace.SessionStateProxy.GetVariable('ansibleTestArgSpec')
 Resolve-CircularReference -Hash $arg_spec
 
-ConvertTo-Json -InputObject $arg_spec -Compress -Depth 99
\ No newline at end of file
+ConvertTo-Json -InputObject $arg_spec -Compress -Depth 99
diff --git a/test/lib/ansible_test/_internal/sanity/integration_aliases.py b/test/lib/ansible_test/_internal/sanity/integration_aliases.py
index a72e6669bf7..18002deaad9 100644
--- a/test/lib/ansible_test/_internal/sanity/integration_aliases.py
+++ b/test/lib/ansible_test/_internal/sanity/integration_aliases.py
@@ -219,11 +219,18 @@ class IntegrationAliasesTest(SanityVersionNeutral):
                     messages.append(SanityMessage('invalid alias `%s`' % alias, '%s/aliases' % target.path))
 
         messages += self.check_ci_group(
-            targets=tuple(filter_targets(posix_targets, ['cloud/'], include=False, directories=False, errors=False)),
+            targets=tuple(filter_targets(posix_targets, ['cloud/', 'shippable/generic/'], include=False,
+                                         directories=False, errors=False)),
             find=self.format_shippable_group_alias('linux').replace('linux', 'posix'),
             find_incidental=['shippable/posix/incidental/'],
         )
 
+        messages += self.check_ci_group(
+            targets=tuple(filter_targets(posix_targets, ['shippable/generic/'], include=True, directories=False,
+                                         errors=False)),
+            find=self.format_shippable_group_alias('generic'),
+        )
+
         for cloud in clouds:
             messages += self.check_ci_group(
                 targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], include=True, directories=False, errors=False)),
diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt
index ff4a4ce1e9b..c165d6bb3b3 100644
--- a/test/sanity/ignore.txt
+++ b/test/sanity/ignore.txt
@@ -270,6 +270,9 @@ test/integration/targets/ansible-runner/files/playbook_example1.py metaclass-boi
 test/integration/targets/ansible-test/ansible_collections/ns/col/plugins/modules/hello.py pylint:relative-beyond-top-level
 test/integration/targets/ansible-test/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py pylint:relative-beyond-top-level
 test/integration/targets/ansible-test/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py pylint:relative-beyond-top-level
+test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py pylint:relative-beyond-top-level
+test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py pylint:relative-beyond-top-level
+test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py pylint:relative-beyond-top-level
 test/integration/targets/async/library/async_test.py future-import-boilerplate
 test/integration/targets/async/library/async_test.py metaclass-boilerplate
 test/integration/targets/async_fail/library/async_test.py future-import-boilerplate
diff --git a/test/utils/shippable/generic.sh b/test/utils/shippable/generic.sh
new file mode 100755
index 00000000000..28eb12688ed
--- /dev/null
+++ b/test/utils/shippable/generic.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o pipefail -eux
+
+declare -a args
+IFS='/:' read -ra args <<< "$1"
+
+python="${args[1]}"
+
+if [ "${#args[@]}" -gt 2 ]; then
+    target="shippable/generic/group${args[2]}/"
+else
+    target="shippable/generic/"
+fi
+
+# shellcheck disable=SC2086
+ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
+    --docker default --python "${python}"