diff --git a/lib/ansible/modules/shippable.yml b/lib/ansible/modules/shippable.yml
index c392bfa675f..a2c4e33585c 100644
--- a/lib/ansible/modules/shippable.yml
+++ b/lib/ansible/modules/shippable.yml
@@ -27,6 +27,8 @@ matrix:
     - env: TEST=integration PLATFORM=freebsd VERSION=10.3-STABLE
 
     - env: TEST=integration PLATFORM=osx VERSION=10.11
+
+    - env: TEST=sanity INSTALL_DEPS=1
 build:
   pre_ci_boot:
     options: "--privileged=false --net=bridge"
diff --git a/lib/ansible/modules/test/utils/shippable/integration.sh b/lib/ansible/modules/test/utils/shippable/integration.sh
index ee16e765c15..cf10e681bfb 100755
--- a/lib/ansible/modules/test/utils/shippable/integration.sh
+++ b/lib/ansible/modules/test/utils/shippable/integration.sh
@@ -10,7 +10,7 @@ repo="${REPO_NAME}"
 
 if [ "${is_pr}" != "true" ]; then
     echo "Module integration tests are only supported on pull requests."
-    exit 1
+    exit 0
 fi
 
 case "${repo}" in
diff --git a/lib/ansible/modules/test/utils/shippable/sanity-skip-python24.txt b/lib/ansible/modules/test/utils/shippable/sanity-skip-python24.txt
new file mode 100644
index 00000000000..1434a04094b
--- /dev/null
+++ b/lib/ansible/modules/test/utils/shippable/sanity-skip-python24.txt
@@ -0,0 +1 @@
+/cloud/
diff --git a/lib/ansible/modules/test/utils/shippable/sanity-skip-python3.txt b/lib/ansible/modules/test/utils/shippable/sanity-skip-python3.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/ansible/modules/test/utils/shippable/sanity-test-python24.txt b/lib/ansible/modules/test/utils/shippable/sanity-test-python24.txt
new file mode 100644
index 00000000000..5ad993ee773
--- /dev/null
+++ b/lib/ansible/modules/test/utils/shippable/sanity-test-python24.txt
@@ -0,0 +1,2 @@
+cloud/amazon/_ec2_ami_search.py
+cloud/amazon/ec2_facts.py
diff --git a/lib/ansible/modules/test/utils/shippable/sanity.sh b/lib/ansible/modules/test/utils/shippable/sanity.sh
new file mode 100755
index 00000000000..5ff826efbae
--- /dev/null
+++ b/lib/ansible/modules/test/utils/shippable/sanity.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -eux
+
+source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))")
+
+install_deps="${INSTALL_DEPS:-}"
+
+cd "${source_root}"
+
+if [ "${install_deps}" != "" ]; then
+    add-apt-repository ppa:fkrull/deadsnakes && apt-get update -qq && apt-get install python2.4 -qq
+
+    pip install git+https://github.com/ansible/ansible.git@devel#egg=ansible
+    pip install git+https://github.com/sivel/ansible-testing.git#egg=ansible_testing
+fi
+
+python2.4 -m compileall -fq   -i                   "test/utils/shippable/sanity-test-python24.txt"
+python2.4 -m compileall -fq   -x "($(printf %s "$(< "test/utils/shippable/sanity-skip-python24.txt"))" | tr '\n' '|')" .
+python2.6 -m compileall -fq .
+python2.7 -m compileall -fq .
+python3.5 -m compileall -fq . -x "($(printf %s "$(< "test/utils/shippable/sanity-skip-python3.txt"))"  | tr '\n' '|')"
+
+ansible-validate-modules --exclude '/utilities/|/shippable(/|$)' .