diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7dbd83908..fcbd40b74 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -56,6 +56,7 @@ jobs:
             - 'pyproject.toml'
             - 'poetry.lock'
             - 'docker/**'
+            - 'scripts-dev/complement.sh'
 
           linting:
             - 'synapse/**'
diff --git a/changelog.d/16466.misc b/changelog.d/16466.misc
new file mode 100644
index 000000000..471056bb0
--- /dev/null
+++ b/changelog.d/16466.misc
@@ -0,0 +1 @@
+Update complement.sh to match new public API shape.
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 8416b5567..3e0cddb52 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -214,7 +214,7 @@ fi
 
 extra_test_args=()
 
-test_tags="synapse_blacklist,msc3874,msc3890,msc3391,msc3930,faster_joins"
+test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902"
 
 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
@@ -277,4 +277,4 @@ export PASS_SYNAPSE_LOG_TESTING=1
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
 
-go test -v -tags $test_tags -count=1 "${extra_test_args[@]}" "$@" ./tests/...
+go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" $test_packages