diff --git a/.buildkite/.env b/.buildkite/.env
index 85b102d07..a2969b96a 100644
--- a/.buildkite/.env
+++ b/.buildkite/.env
@@ -7,7 +7,6 @@ BUILDKITE_JOB_ID
 BUILDKITE_BUILD_URL
 BUILDKITE_PROJECT_SLUG
 BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
 BUILDKITE_TAG
 CODECOV_TOKEN
 TRIAL_FLAGS
diff --git a/.buildkite/merge_base_branch.sh b/.buildkite/merge_base_branch.sh
deleted file mode 100755
index 361440fd1..000000000
--- a/.buildkite/merge_base_branch.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
-
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="develop"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
-fi
-
-echo "--- merge_base_branch $GITBASE"
-
-# Show what we are before
-git --no-pager show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
-
-# Show what we are after.
-git --no-pager show -s
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8612d1fb3..5349e8313 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -47,11 +47,9 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v2
       - run: pip install tox
-      - name: Patch Buildkite-specific test script
-        run: |
-          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
-            scripts-dev/check-newsfragment
       - run: scripts-dev/check-newsfragment
+        env:
+          PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
   lint-sdist:
     runs-on: ubuntu-latest
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index af6d32e33..393a548d5 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -11,7 +11,7 @@ set -e
 git remote set-branches --add origin develop
 git fetch -q origin develop
 
-pr="$BUILDKITE_PULL_REQUEST"
+pr="$PULL_REQUEST_NUMBER"
 
 # if there are changes in the debian directory, check that the debian changelog
 # has been updated