diff --git a/lib/ansible/modules/shippable.yml b/lib/ansible/modules/shippable.yml
index a2c4e33585c..a8dd0fc5a9f 100644
--- a/lib/ansible/modules/shippable.yml
+++ b/lib/ansible/modules/shippable.yml
@@ -29,6 +29,8 @@ matrix:
     - env: TEST=integration PLATFORM=osx VERSION=10.11
 
     - env: TEST=sanity INSTALL_DEPS=1
+
+    - env: TEST=docs
 build:
   pre_ci_boot:
     options: "--privileged=false --net=bridge"
diff --git a/lib/ansible/modules/test/utils/shippable/docs-requirements.txt b/lib/ansible/modules/test/utils/shippable/docs-requirements.txt
new file mode 100644
index 00000000000..4e859bb8c71
--- /dev/null
+++ b/lib/ansible/modules/test/utils/shippable/docs-requirements.txt
@@ -0,0 +1,2 @@
+jinja2
+pyyaml
diff --git a/lib/ansible/modules/test/utils/shippable/docs.sh b/lib/ansible/modules/test/utils/shippable/docs.sh
new file mode 100755
index 00000000000..9b5a6164f64
--- /dev/null
+++ b/lib/ansible/modules/test/utils/shippable/docs.sh
@@ -0,0 +1,55 @@
+#!/bin/bash -eux
+
+set -o pipefail
+
+ansible_repo_url="https://github.com/ansible/ansible.git"
+
+build_dir="${SHIPPABLE_BUILD_DIR}"
+repo="${REPO_NAME}"
+
+case "${repo}" in
+    "ansible-modules-core")
+        this_module_group="core"
+        other_module_group="extras"
+        ;;
+    "ansible-modules-extras")
+        this_module_group="extras"
+        other_module_group="core"
+        ;;
+    *)
+        echo "Unsupported repo name: ${repo}"
+        exit 1
+        ;;
+esac
+
+modules_tmp_dir="${build_dir}.tmp"
+this_modules_dir="${build_dir}/lib/ansible/modules/${this_module_group}"
+other_modules_dir="${build_dir}/lib/ansible/modules/${other_module_group}"
+
+cd /
+mv "${build_dir}" "${modules_tmp_dir}"
+git clone "${ansible_repo_url}" "${build_dir}"
+cd "${build_dir}"
+rmdir "${this_modules_dir}"
+mv "${modules_tmp_dir}" "${this_modules_dir}"
+mv "${this_modules_dir}/shippable" "${build_dir}"
+git submodule init "${other_modules_dir}"
+git submodule sync "${other_modules_dir}"
+git submodule update "${other_modules_dir}"
+
+pip install -r lib/ansible/modules/${this_module_group}/test/utils/shippable/docs-requirements.txt --upgrade
+pip list
+
+source hacking/env-setup
+
+PAGER=/bin/cat \
+    ANSIBLE_DEPRECATION_WARNINGS=false \
+    bin/ansible-doc -l \
+    2>/tmp/ansible-doc.err
+
+if [ -s /tmp/ansible-doc.err ]; then
+    # report warnings as errors
+    echo "Output from 'ansible-doc -l' on stderr is considered an error:"
+    cat /tmp/ansible-doc.err
+    exit 1
+fi