diff --git a/lib/ansible/modules/extras/shippable.yml b/lib/ansible/modules/extras/shippable.yml
index c392bfa675f..a2c4e33585c 100644
--- a/lib/ansible/modules/extras/shippable.yml
+++ b/lib/ansible/modules/extras/shippable.yml
@@ -27,6 +27,8 @@ matrix:
     - env: TEST=integration PLATFORM=freebsd VERSION=10.3-STABLE
 
     - env: TEST=integration PLATFORM=osx VERSION=10.11
+
+    - env: TEST=sanity INSTALL_DEPS=1
 build:
   pre_ci_boot:
     options: "--privileged=false --net=bridge"
diff --git a/lib/ansible/modules/extras/test/utils/shippable/integration.sh b/lib/ansible/modules/extras/test/utils/shippable/integration.sh
index ee16e765c15..cf10e681bfb 100755
--- a/lib/ansible/modules/extras/test/utils/shippable/integration.sh
+++ b/lib/ansible/modules/extras/test/utils/shippable/integration.sh
@@ -10,7 +10,7 @@ repo="${REPO_NAME}"
 
 if [ "${is_pr}" != "true" ]; then
     echo "Module integration tests are only supported on pull requests."
-    exit 1
+    exit 0
 fi
 
 case "${repo}" in
diff --git a/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python24.txt b/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python24.txt
new file mode 100644
index 00000000000..5e3e5afa8d8
--- /dev/null
+++ b/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python24.txt
@@ -0,0 +1,14 @@
+/cloud/
+/clustering/consul.*.py
+/clustering/znode.py
+/database/influxdb/
+/database/mssql/
+/monitoring/zabbix.*.py
+/network/f5/
+/notification/pushbullet.py
+/packaging/language/maven_artifact.py
+/packaging/os/dnf.py
+/packaging/os/layman.py
+/remote_management/ipmi/
+/univention/
+/web_infrastructure/letsencrypt.py
diff --git a/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python3.txt b/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python3.txt
new file mode 100644
index 00000000000..465597ae676
--- /dev/null
+++ b/lib/ansible/modules/extras/test/utils/shippable/sanity-skip-python3.txt
@@ -0,0 +1,95 @@
+/cloud/amazon/cloudtrail.py
+/cloud/amazon/dynamodb_table.py
+/cloud/amazon/ec2_ami_copy.py
+/cloud/amazon/ec2_customer_gateway.py
+/cloud/amazon/ec2_elb_facts.py
+/cloud/amazon/ec2_eni.py
+/cloud/amazon/ec2_eni_facts.py
+/cloud/amazon/ec2_remote_facts.py
+/cloud/amazon/ec2_snapshot_facts.py
+/cloud/amazon/ec2_vol_facts.py
+/cloud/amazon/ec2_vpc_dhcp_options_facts.py
+/cloud/amazon/ec2_vpc_igw.py
+/cloud/amazon/ec2_vpc_nacl.py
+/cloud/amazon/ec2_vpc_net_facts.py
+/cloud/amazon/ec2_vpc_route_table.py
+/cloud/amazon/ec2_vpc_route_table_facts.py
+/cloud/amazon/ec2_vpc_subnet.py
+/cloud/amazon/ec2_vpc_subnet_facts.py
+/cloud/amazon/ecs_cluster.py
+/cloud/amazon/ecs_service.py
+/cloud/amazon/ecs_service_facts.py
+/cloud/amazon/ecs_task.py
+/cloud/amazon/ecs_taskdefinition.py
+/cloud/amazon/route53_facts.py
+/cloud/amazon/route53_health_check.py
+/cloud/amazon/route53_zone.py
+/cloud/amazon/s3_bucket.py
+/cloud/amazon/s3_lifecycle.py
+/cloud/amazon/s3_logging.py
+/cloud/amazon/sns_topic.py
+/cloud/amazon/sqs_queue.py
+/cloud/amazon/sts_assume_role.py
+/cloud/amazon/sts_session_token.py
+/cloud/centurylink/clc_aa_policy.py
+/cloud/centurylink/clc_group.py
+/cloud/centurylink/clc_publicip.py
+/cloud/google/gce_img.py
+/cloud/google/gce_tag.py
+/cloud/misc/ovirt.py
+/cloud/misc/proxmox.py
+/cloud/misc/proxmox_template.py
+/cloud/misc/virt.py
+/cloud/misc/virt_net.py
+/cloud/misc/virt_pool.py
+/cloud/profitbricks/profitbricks.py
+/cloud/profitbricks/profitbricks_volume.py
+/cloud/rackspace/rax_clb_ssl.py
+/cloud/xenserver_facts.py
+/clustering/consul.py
+/clustering/consul_acl.py
+/clustering/consul_kv.py
+/clustering/consul_session.py
+/commands/expect.py
+/database/misc/mongodb_parameter.py
+/database/misc/mongodb_user.py
+/database/misc/redis.py
+/database/mysql/mysql_replication.py
+/database/postgresql/postgresql_ext.py
+/database/postgresql/postgresql_lang.py
+/database/vertica/vertica_configuration.py
+/database/vertica/vertica_facts.py
+/database/vertica/vertica_role.py
+/database/vertica/vertica_schema.py
+/database/vertica/vertica_user.py
+/monitoring/bigpanda.py
+/monitoring/boundary_meter.py
+/monitoring/circonus_annotation.py
+/monitoring/datadog_monitor.py
+/monitoring/rollbar_deployment.py
+/monitoring/sensu_check.py
+/monitoring/stackdriver.py
+/monitoring/zabbix_group.py
+/monitoring/zabbix_host.py
+/monitoring/zabbix_hostmacro.py
+/monitoring/zabbix_screen.py
+/network/citrix/netscaler.py
+/network/cloudflare_dns.py
+/network/dnsimple.py
+/network/dnsmadeeasy.py
+/network/f5/bigip_facts.py
+/network/f5/bigip_gtm_virtual_server.py
+/network/f5/bigip_gtm_wide_ip.py
+/network/f5/bigip_monitor_http.py
+/network/f5/bigip_monitor_tcp.py
+/network/f5/bigip_node.py
+/network/f5/bigip_pool.py
+/network/f5/bigip_pool_member.py
+/network/f5/bigip_virtual_server.py
+/network/nmcli.py
+/network/openvswitch_bridge.py
+/network/openvswitch_port.py
+/notification/irc.py
+/notification/jabber.py
+/notification/mail.py
+/notification/mqtt.py
diff --git a/lib/ansible/modules/extras/test/utils/shippable/sanity-test-python24.txt b/lib/ansible/modules/extras/test/utils/shippable/sanity-test-python24.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/ansible/modules/extras/test/utils/shippable/sanity.sh b/lib/ansible/modules/extras/test/utils/shippable/sanity.sh
new file mode 100755
index 00000000000..5ff826efbae
--- /dev/null
+++ b/lib/ansible/modules/extras/test/utils/shippable/sanity.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -eux
+
+source_root=$(python -c "from os import path; print(path.abspath(path.join(path.dirname('$0'), '../../..')))")
+
+install_deps="${INSTALL_DEPS:-}"
+
+cd "${source_root}"
+
+if [ "${install_deps}" != "" ]; then
+    add-apt-repository ppa:fkrull/deadsnakes && apt-get update -qq && apt-get install python2.4 -qq
+
+    pip install git+https://github.com/ansible/ansible.git@devel#egg=ansible
+    pip install git+https://github.com/sivel/ansible-testing.git#egg=ansible_testing
+fi
+
+python2.4 -m compileall -fq   -i                   "test/utils/shippable/sanity-test-python24.txt"
+python2.4 -m compileall -fq   -x "($(printf %s "$(< "test/utils/shippable/sanity-skip-python24.txt"))" | tr '\n' '|')" .
+python2.6 -m compileall -fq .
+python2.7 -m compileall -fq .
+python3.5 -m compileall -fq . -x "($(printf %s "$(< "test/utils/shippable/sanity-skip-python3.txt"))"  | tr '\n' '|')"
+
+ansible-validate-modules --exclude '/utilities/|/shippable(/|$)' .