Ensure that testing containers happens as part of a pulumi publish event (#6030)
This commit is contained in:
Paul Stack
GitHub
parent
2c6296add2
commit
fead88d4a6
25
.github/workflows/master.yml
vendored
25
.github/workflows/master.yml
vendored
|
|
@ -75,7 +75,7 @@ jobs:
|
|||
publish-binaries:
|
||||
name: Publish Binaries
|
||||
runs-on: macos-latest
|
||||
needs: build-and-test
|
||||
needs: [build-and-test, windows-build]
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [ 1.15.x ]
|
||||
|
|
@ -277,5 +277,28 @@ jobs:
|
|||
run: |
|
||||
cd src\github.com\${{ github.repository }}
|
||||
dotnet msbuild /t:Publish /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi"
|
||||
verify-containers:
|
||||
name: Run Container Tests
|
||||
needs: [publish-binaries, publish-sdks]
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [ ubuntu-latest, macos-latest ]
|
||||
go-version: [1.15.x]
|
||||
python-version: [ 3.9.x ]
|
||||
dotnet-version: [ 3.1.x ]
|
||||
node-version: [ 14.x ]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
- name: Set up Go ${{ matrix.go-version }}
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Fetch Tags
|
||||
run: |
|
||||
git fetch --quiet --prune --unshallow --tags
|
||||
- name: Run Container Tests
|
||||
run: make test_containers VERSION=v$(pulumictl get version --language generic -o)
|
||||
|
||||
|
||||
|
|
|
|||
105
.github/workflows/prerelease.yml
vendored
105
.github/workflows/prerelease.yml
vendored
|
|
@ -73,7 +73,7 @@ jobs:
|
|||
publish-binaries:
|
||||
name: Publish Binaries
|
||||
runs-on: macos-latest
|
||||
needs: build-and-test
|
||||
needs: [build-and-test, windows-release]
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [ 1.15.x ]
|
||||
|
|
@ -190,5 +190,104 @@ jobs:
|
|||
PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
|
||||
PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget
|
||||
PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
|
||||
|
||||
|
||||
windows-release:
|
||||
name: Windows Build + Release
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [ 1.15.x ]
|
||||
node-version: [ 14.x ]
|
||||
python-version: [ 3.9.x ]
|
||||
dotnet: [ 3.1.x ]
|
||||
runs-on: windows-latest
|
||||
env:
|
||||
GOPATH: ${{ github.workspace }}
|
||||
PULUMI_LOCAL_NUGET: "D:\\Pulumi\\nuget"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_LEGACY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_LEGACY }}
|
||||
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
|
||||
steps:
|
||||
- name: Install DotNet ${{ matrix.dotnet }}
|
||||
uses: actions/setup-dotnet@v1
|
||||
with:
|
||||
dotnet-version: ${{ matrix.dotnet }}
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v1
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
- name: Set up Go ${{ matrix.go-version }}
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Clean
|
||||
run: dotnet nuget locals all --clear
|
||||
- name: Install Python Deps
|
||||
run: |
|
||||
pip3 install pyenv-win
|
||||
pip3 install pipenv
|
||||
- name: Set Build Env Vars
|
||||
shell: bash
|
||||
run: |
|
||||
echo "D:\\Pulumi\\bin" >> $GITHUB_PATH
|
||||
- name: Install AWS CLI Tools
|
||||
uses: chrislennon/action-aws-cli@v1.1
|
||||
- name: Check out scripts repo into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
repository: pulumi/scripts
|
||||
path: ./src/github.com/pulumi/scripts
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
path: ./src/github.com/${{ github.repository }}
|
||||
- name: Fetch Tags
|
||||
run: |
|
||||
cd ./src/github.com/${{ github.repository }} && git fetch --quiet --prune --unshallow --tags
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
cd src\github.com\${{ github.repository }}
|
||||
pushd .
|
||||
cd sdk
|
||||
go mod tidy
|
||||
go mod download
|
||||
popd
|
||||
pushd .
|
||||
cd pkg
|
||||
go mod tidy
|
||||
go mod download
|
||||
popd
|
||||
pushd .
|
||||
cd tests
|
||||
go mod tidy
|
||||
go mod download
|
||||
popd
|
||||
- name: Publish Binary
|
||||
run: |
|
||||
cd src\github.com\${{ github.repository }}
|
||||
dotnet msbuild /t:Publish /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi"
|
||||
verify-containers:
|
||||
name: Run Container Tests
|
||||
needs: [publish-binaries, publish-sdks]
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [ ubuntu-latest, macos-latest ]
|
||||
go-version: [1.15.x]
|
||||
python-version: [ 3.9.x ]
|
||||
dotnet-version: [ 3.1.x ]
|
||||
node-version: [ 14.x ]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
- name: Set up Go ${{ matrix.go-version }}
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Fetch Tags
|
||||
run: |
|
||||
git fetch --quiet --prune --unshallow --tags
|
||||
- name: Run Container Tests
|
||||
run: make test_containers VERSION=v$(pulumictl get version --language generic -o)
|
||||
|
|
|
|||
29
.github/workflows/release.yml
vendored
29
.github/workflows/release.yml
vendored
|
|
@ -68,7 +68,7 @@ jobs:
|
|||
docker:
|
||||
name: Build Slim Docker Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: publish-sdks
|
||||
needs: [publish-sdks, verify-containers]
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -169,7 +169,7 @@ jobs:
|
|||
publish-binaries:
|
||||
name: Publish Binaries
|
||||
runs-on: macos-latest
|
||||
needs: build-and-test
|
||||
needs: [build-and-test, windows-release]
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [ 1.15.x ]
|
||||
|
|
@ -364,5 +364,26 @@ jobs:
|
|||
run: |
|
||||
cd src\github.com\${{ github.repository }}
|
||||
dotnet msbuild /t:ReleaseProcess /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi"
|
||||
|
||||
|
||||
verify-containers:
|
||||
name: Run Container Tests
|
||||
needs: [publish-binaries, publish-sdks]
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [ ubuntu-latest, macos-latest ]
|
||||
go-version: [1.15.x]
|
||||
python-version: [ 3.9.x ]
|
||||
dotnet-version: [ 3.1.x ]
|
||||
node-version: [ 14.x ]
|
||||
runs-on: ${{ matrix.platform }}
|
||||
steps:
|
||||
- name: Set up Go ${{ matrix.go-version }}
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: ${{ matrix.go-version }}
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Fetch Tags
|
||||
run: |
|
||||
git fetch --quiet --prune --unshallow --tags
|
||||
- name: Run Container Tests
|
||||
run: make test_containers VERSION=v$(pulumictl get version --language generic -o)
|
||||
|
|
|
|||
13
Makefile
13
Makefile
|
|
@ -88,17 +88,10 @@ publish_packages:
|
|||
$(call STEP_MESSAGE)
|
||||
./scripts/publish_packages.sh
|
||||
|
||||
# Run the integration tests for our DockerHub containers. We do so only via the
|
||||
# "Travis Cron" job type, because (1) the tests can only be ran _after_ we publish
|
||||
# the current SDK version, since it is required by the Docker build. And (2) the
|
||||
# tests (currently) aren't reliable enough to run as part of every push to master.
|
||||
#
|
||||
# So instead we run the ~daily on master. Where we know the current SDK version
|
||||
# will have been published.
|
||||
.PHONY: test_containers_cron
|
||||
test_containers_cron:
|
||||
.PHONY: test_containers
|
||||
test_containers:
|
||||
$(call STEP_MESSAGE)
|
||||
./scripts/build-docker.sh ${VERSION} --test
|
||||
./scripts/test-containers.sh ${VERSION}
|
||||
|
||||
# The travis_* targets are entrypoints for CI.
|
||||
.PHONY: travis_cron travis_push travis_pull_request travis_api
|
||||
|
|
|
|||
|
|
@ -1,118 +0,0 @@
|
|||
# Clean up the CLI version for dev builds, since they aren't valid tag names.
|
||||
# #!/bin/bash
|
||||
#
|
||||
# Builds the Pulumi docker containers locally. Optionally running tests or
|
||||
# publishing to a container registry.
|
||||
#
|
||||
# Usage: build-docker cli-version [--test] [--publish]
|
||||
set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
readonly SCRIPT_DIR="$( cd "$( dirname "${0}" )" && pwd )"
|
||||
readonly ROOT=${SCRIPT_DIR}/..
|
||||
|
||||
if [ -z "${1:-}" ]; then
|
||||
>&2 echo "error: missing version to publish"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanitize the name of the version, e.g.
|
||||
# "v1.14.0-alpha.1586190504+gf4e9f7e2" -> "v1.14.0-alpha.1586190504".
|
||||
readonly CLI_VERSION="$(echo "${1}" | sed 's/\+.*//g')"
|
||||
|
||||
# The Docker containers built/tested/published from this repository.
|
||||
readonly PULUMI_CONTAINERS=("pulumi" "actions")
|
||||
|
||||
echo_header() {
|
||||
echo -e "\n\033[0;35m${1}\033[0m"
|
||||
}
|
||||
|
||||
test_containers() {
|
||||
echo_header "Executing container runtime tests"
|
||||
|
||||
# Run the container tests, note that we also build the binaries into /tmp for the next step.
|
||||
TEST_BIN_DIRECTORY=$(mktemp -d)
|
||||
pushd ${ROOT}/tests
|
||||
GOOS=linux go test -c -o ${TEST_BIN_DIRECTORY} ${ROOT}/tests/containers/...
|
||||
popd
|
||||
|
||||
# Run tests _within_ the "pulumi" container, ensuring that the CLI is installed
|
||||
# and working correctly.
|
||||
docker run -e RUN_CONTAINER_TESTS=true \
|
||||
-e PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN} \
|
||||
--volume ${TEST_BIN_DIRECTORY}:/src \
|
||||
--entrypoint /bin/bash \
|
||||
pulumi/pulumi:latest \
|
||||
-c "pip install pipenv && /src/pulumi-test-containers -test.parallel=1 -test.v -test.run TestPulumiDockerImage"
|
||||
|
||||
# The actions container should fetch program dependencies from NPM, PIP, etc. before
|
||||
# executing. These tests just shell out to docker run to confirm that.
|
||||
echo_header "Executing container entrypoint tests"
|
||||
pushd ${ROOT}/tests/containers
|
||||
RUN_CONTAINER_TESTS=true go test . -test.run TestPulumiActionsImage -test.v
|
||||
popd
|
||||
}
|
||||
|
||||
# Publishes the built containers to Docker Hub.
|
||||
publish_containers() {
|
||||
echo_header "Publishing containers"
|
||||
|
||||
# Required environment variables.
|
||||
if [ -z "${DOCKER_HUB_USER:-}" ]; then
|
||||
>&2 echo "error: 'DOCKER_HUB_USER' should be defined"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${DOCKER_HUB_PASSWORD:-}" ]; then
|
||||
>&2 echo "error: 'DOCKER_HUB_PASSWORD' should be defined"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# We only want to push docker images for stable versions of Pulumi. So if there is a -alpha
|
||||
# pre-release tag, skip publishing.
|
||||
if [[ "${CLI_VERSION}" == *-alpha* ]]; then
|
||||
>&2 echo "Skipping docker publishing for ${CLI_VERSION} since it is a pre-release"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_PASSWORD}"
|
||||
|
||||
for container in ${PULUMI_CONTAINERS[@]}; do
|
||||
echo "- pulumi/${container}"
|
||||
docker push "pulumi/${container}:${CLI_VERSION}"
|
||||
docker push "pulumi/${container}:latest"
|
||||
done
|
||||
|
||||
docker logout
|
||||
|
||||
# This publishes the SDK specific containers and uses a dispatch event to trigger a GitHub Action
|
||||
pulumictl create containers "${CLI_VERSION//v}"
|
||||
pulumictl create choco-deploy "${CLI_VERSION//v}"
|
||||
}
|
||||
|
||||
echo_header "Building Pulumi containers (${CLI_VERSION})"
|
||||
for container in ${PULUMI_CONTAINERS[@]}; do
|
||||
echo "- Building pulumi/${container}"
|
||||
docker build --build-arg PULUMI_VERSION="${CLI_VERSION}" \
|
||||
-t "pulumi/${container}:${CLI_VERSION}" \
|
||||
-t "pulumi/${container}:latest" \
|
||||
"${SCRIPT_DIR}/../dist/${container}"
|
||||
done
|
||||
|
||||
# Loop through the remaining args, running them in order.
|
||||
for script_arg in "${@:2}"; do
|
||||
case ${script_arg} in
|
||||
"--test")
|
||||
test_containers
|
||||
;;
|
||||
"--publish")
|
||||
echo "Publishing..."
|
||||
publish_containers
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unrecognized argument '${script_arg}'"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
60
scripts/test-containers.sh
Executable file
60
scripts/test-containers.sh
Executable file
|
|
@ -0,0 +1,60 @@
|
|||
# Clean up the CLI version for dev builds, since they aren't valid tag names.
|
||||
# #!/bin/bash
|
||||
#
|
||||
# Builds the Pulumi docker containers locally. Optionally running tests or
|
||||
# publishing to a container registry.
|
||||
#
|
||||
# Usage: build-docker cli-version [--test] [--publish]
|
||||
set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
readonly SCRIPT_DIR="$( cd "$( dirname "${0}" )" && pwd )"
|
||||
readonly ROOT=${SCRIPT_DIR}/..
|
||||
|
||||
if [ -z "${1:-}" ]; then
|
||||
>&2 echo "error: missing version to publish"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanitize the name of the version, e.g.
|
||||
# "v1.14.0-alpha.1586190504+gf4e9f7e2" -> "v1.14.0-alpha.1586190504".
|
||||
readonly CLI_VERSION="$(echo "${1}" | sed 's/\+.*//g')"
|
||||
|
||||
# The Docker containers built/tested/published from this repository.
|
||||
readonly PULUMI_CONTAINERS=("pulumi" "actions")
|
||||
|
||||
echo_header() {
|
||||
echo -e "\n\033[0;35m${1}\033[0m"
|
||||
}
|
||||
|
||||
echo_header "Building local copy of Pulumi containers (${CLI_VERSION})"
|
||||
for container in ${PULUMI_CONTAINERS[@]}; do
|
||||
echo "- Building pulumi/${container}"
|
||||
docker build --build-arg PULUMI_VERSION="${CLI_VERSION}" \
|
||||
-t "pulumi/${container}:${CLI_VERSION}" \
|
||||
-t "pulumi/${container}:latest" \
|
||||
"${SCRIPT_DIR}/../docker/${container}"
|
||||
done
|
||||
echo_header "Executing container runtime tests"
|
||||
|
||||
# Run the container tests, note that we also build the binaries into /tmp for the next step.
|
||||
pushd ${ROOT}/tests
|
||||
GOOS=linux go test -c -o /tmp/pulumi-test-containers ./containers/...
|
||||
popd
|
||||
|
||||
# Run tests _within_ the "pulumi" container, ensuring that the CLI is installed
|
||||
# and working correctly.
|
||||
docker run -e RUN_CONTAINER_TESTS=true \
|
||||
-e PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN} \
|
||||
--volume /tmp:/src \
|
||||
--entrypoint /bin/bash \
|
||||
pulumi/pulumi:latest \
|
||||
-c "pip install pipenv && /src/pulumi-test-containers -test.parallel=1 -test.timeout=1h -test.v -test.run TestPulumiDockerImage"
|
||||
|
||||
# The actions container should fetch program dependencies from NPM, PIP, etc. before
|
||||
# executing. These tests just shell out to docker run to confirm that.
|
||||
echo_header "Executing container entrypoint tests"
|
||||
pushd ${ROOT}/tests/containers
|
||||
RUN_CONTAINER_TESTS=true go test . -test.run TestPulumiActionsImage -test.v -test.timeout=1h
|
||||
popd
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/bin/bash
|
||||
# update_homebrew.sh uses `brew bump-formula-pr` to update the formula for the Pulumi CLI and SDKs
|
||||
set -o nounset
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
readonly ROOT=$(dirname "${0}")/..
|
||||
|
||||
if [[ "${TRAVIS:-}" != "true" ]]; then
|
||||
echo "error: this script should be run from within Travis"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${PULUMI_BOT_GITHUB_API_TOKEN:-}" ]]; then
|
||||
echo "error: PULUMI_BOT_GITHUB_API_TOKEN must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! echo "${TRAVIS_TAG:-}" | grep -q -e "^v[0-9]\+\.[0-9]\+\.[0-9]\+$"; then
|
||||
echo "Skipping Homebrew formula update; ${TRAVIS_TAG:-} does not denote a released version"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${TRAVIS_OS_NAME:-}" != "osx" ]]; then
|
||||
echo "Skipping Homebrew formula updte; not running on OS X"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
HOMEBREW_GITHUB_API_TOKEN="${PULUMI_BOT_GITHUB_API_TOKEN:-}" brew bump-formula-pr --tag="${TRAVIS_TAG:-}" --revision="${TRAVIS_COMMIT:-}" pulumi
|
||||
exit 0
|
||||
Loading…
Reference in a new issue