diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0901d0e83..8d5da28bd 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -75,7 +75,7 @@ jobs: publish-binaries: name: Publish Binaries runs-on: macos-latest - needs: build-and-test + needs: [build-and-test, windows-build] strategy: matrix: go-version: [ 1.15.x ] @@ -277,5 +277,28 @@ jobs: run: | cd src\github.com\${{ github.repository }} dotnet msbuild /t:Publish /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi" + verify-containers: + name: Run Container Tests + needs: [publish-binaries, publish-sdks] + strategy: + matrix: + platform: [ ubuntu-latest, macos-latest ] + go-version: [1.15.x] + python-version: [ 3.9.x ] + dotnet-version: [ 3.1.x ] + node-version: [ 14.x ] + runs-on: ${{ matrix.platform }} + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Run Container Tests + run: make test_containers VERSION=v$(pulumictl get version --language generic -o) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index c01b97785..92e95340d 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -73,7 +73,7 @@ jobs: publish-binaries: name: Publish Binaries runs-on: macos-latest - needs: build-and-test + needs: [build-and-test, windows-release] strategy: matrix: go-version: [ 1.15.x ] @@ -190,5 +190,104 @@ jobs: PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi - - + windows-release: + name: Windows Build + Release + strategy: + matrix: + go-version: [ 1.15.x ] + node-version: [ 14.x ] + python-version: [ 3.9.x ] + dotnet: [ 3.1.x ] + runs-on: windows-latest + env: + GOPATH: ${{ github.workspace }} + PULUMI_LOCAL_NUGET: "D:\\Pulumi\\nuget" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_LEGACY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_LEGACY }} + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + steps: + - name: Install DotNet ${{ matrix.dotnet }} + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Clean + run: dotnet nuget locals all --clear + - name: Install Python Deps + run: | + pip3 install pyenv-win + pip3 install pipenv + - name: Set Build Env Vars + shell: bash + run: | + echo "D:\\Pulumi\\bin" >> $GITHUB_PATH + - name: Install AWS CLI Tools + uses: chrislennon/action-aws-cli@v1.1 + - name: Check out scripts repo into the Go module directory + uses: actions/checkout@v2 + with: + repository: pulumi/scripts + path: ./src/github.com/pulumi/scripts + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + path: ./src/github.com/${{ github.repository }} + - name: Fetch Tags + run: | + cd ./src/github.com/${{ github.repository }} && git fetch --quiet --prune --unshallow --tags + - name: Get dependencies + run: | + cd src\github.com\${{ github.repository }} + pushd . + cd sdk + go mod tidy + go mod download + popd + pushd . + cd pkg + go mod tidy + go mod download + popd + pushd . + cd tests + go mod tidy + go mod download + popd + - name: Publish Binary + run: | + cd src\github.com\${{ github.repository }} + dotnet msbuild /t:Publish /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi" + verify-containers: + name: Run Container Tests + needs: [publish-binaries, publish-sdks] + strategy: + matrix: + platform: [ ubuntu-latest, macos-latest ] + go-version: [1.15.x] + python-version: [ 3.9.x ] + dotnet-version: [ 3.1.x ] + node-version: [ 14.x ] + runs-on: ${{ matrix.platform }} + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Run Container Tests + run: make test_containers VERSION=v$(pulumictl get version --language generic -o) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b628c5a17..079abd888 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: docker: name: Build Slim Docker Images runs-on: ubuntu-latest - needs: publish-sdks + needs: [publish-sdks, verify-containers] steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -169,7 +169,7 @@ jobs: publish-binaries: name: Publish Binaries runs-on: macos-latest - needs: build-and-test + needs: [build-and-test, windows-release] strategy: matrix: go-version: [ 1.15.x ] @@ -364,5 +364,26 @@ jobs: run: | cd src\github.com\${{ github.repository }} dotnet msbuild /t:ReleaseProcess /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi" - - + verify-containers: + name: Run Container Tests + needs: [publish-binaries, publish-sdks] + strategy: + matrix: + platform: [ ubuntu-latest, macos-latest ] + go-version: [1.15.x] + python-version: [ 3.9.x ] + dotnet-version: [ 3.1.x ] + node-version: [ 14.x ] + runs-on: ${{ matrix.platform }} + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Run Container Tests + run: make test_containers VERSION=v$(pulumictl get version --language generic -o) diff --git a/Makefile b/Makefile index 80836a6ae..b7c6bb489 100644 --- a/Makefile +++ b/Makefile @@ -88,17 +88,10 @@ publish_packages: $(call STEP_MESSAGE) ./scripts/publish_packages.sh -# Run the integration tests for our DockerHub containers. We do so only via the -# "Travis Cron" job type, because (1) the tests can only be ran _after_ we publish -# the current SDK version, since it is required by the Docker build. And (2) the -# tests (currently) aren't reliable enough to run as part of every push to master. -# -# So instead we run the ~daily on master. Where we know the current SDK version -# will have been published. -.PHONY: test_containers_cron -test_containers_cron: +.PHONY: test_containers +test_containers: $(call STEP_MESSAGE) - ./scripts/build-docker.sh ${VERSION} --test + ./scripts/test-containers.sh ${VERSION} # The travis_* targets are entrypoints for CI. .PHONY: travis_cron travis_push travis_pull_request travis_api diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh deleted file mode 100755 index d4b3d1aa4..000000000 --- a/scripts/build-docker.sh +++ /dev/null @@ -1,118 +0,0 @@ -# Clean up the CLI version for dev builds, since they aren't valid tag names. -# #!/bin/bash -# -# Builds the Pulumi docker containers locally. Optionally running tests or -# publishing to a container registry. -# -# Usage: build-docker cli-version [--test] [--publish] -set -o nounset -set -o errexit -set -o pipefail - -readonly SCRIPT_DIR="$( cd "$( dirname "${0}" )" && pwd )" -readonly ROOT=${SCRIPT_DIR}/.. - -if [ -z "${1:-}" ]; then - >&2 echo "error: missing version to publish" - exit 1 -fi - -# Sanitize the name of the version, e.g. -# "v1.14.0-alpha.1586190504+gf4e9f7e2" -> "v1.14.0-alpha.1586190504". -readonly CLI_VERSION="$(echo "${1}" | sed 's/\+.*//g')" - -# The Docker containers built/tested/published from this repository. -readonly PULUMI_CONTAINERS=("pulumi" "actions") - -echo_header() { - echo -e "\n\033[0;35m${1}\033[0m" -} - -test_containers() { - echo_header "Executing container runtime tests" - - # Run the container tests, note that we also build the binaries into /tmp for the next step. - TEST_BIN_DIRECTORY=$(mktemp -d) - pushd ${ROOT}/tests - GOOS=linux go test -c -o ${TEST_BIN_DIRECTORY} ${ROOT}/tests/containers/... - popd - - # Run tests _within_ the "pulumi" container, ensuring that the CLI is installed - # and working correctly. - docker run -e RUN_CONTAINER_TESTS=true \ - -e PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN} \ - --volume ${TEST_BIN_DIRECTORY}:/src \ - --entrypoint /bin/bash \ - pulumi/pulumi:latest \ - -c "pip install pipenv && /src/pulumi-test-containers -test.parallel=1 -test.v -test.run TestPulumiDockerImage" - - # The actions container should fetch program dependencies from NPM, PIP, etc. before - # executing. These tests just shell out to docker run to confirm that. - echo_header "Executing container entrypoint tests" - pushd ${ROOT}/tests/containers - RUN_CONTAINER_TESTS=true go test . -test.run TestPulumiActionsImage -test.v - popd -} - -# Publishes the built containers to Docker Hub. -publish_containers() { - echo_header "Publishing containers" - - # Required environment variables. - if [ -z "${DOCKER_HUB_USER:-}" ]; then - >&2 echo "error: 'DOCKER_HUB_USER' should be defined" - exit 1 - fi - - if [ -z "${DOCKER_HUB_PASSWORD:-}" ]; then - >&2 echo "error: 'DOCKER_HUB_PASSWORD' should be defined" - exit 1 - fi - - # We only want to push docker images for stable versions of Pulumi. So if there is a -alpha - # pre-release tag, skip publishing. - if [[ "${CLI_VERSION}" == *-alpha* ]]; then - >&2 echo "Skipping docker publishing for ${CLI_VERSION} since it is a pre-release" - exit 0 - fi - - docker login -u "${DOCKER_HUB_USER}" -p "${DOCKER_HUB_PASSWORD}" - - for container in ${PULUMI_CONTAINERS[@]}; do - echo "- pulumi/${container}" - docker push "pulumi/${container}:${CLI_VERSION}" - docker push "pulumi/${container}:latest" - done - - docker logout - - # This publishes the SDK specific containers and uses a dispatch event to trigger a GitHub Action - pulumictl create containers "${CLI_VERSION//v}" - pulumictl create choco-deploy "${CLI_VERSION//v}" -} - -echo_header "Building Pulumi containers (${CLI_VERSION})" -for container in ${PULUMI_CONTAINERS[@]}; do - echo "- Building pulumi/${container}" - docker build --build-arg PULUMI_VERSION="${CLI_VERSION}" \ - -t "pulumi/${container}:${CLI_VERSION}" \ - -t "pulumi/${container}:latest" \ - "${SCRIPT_DIR}/../dist/${container}" -done - -# Loop through the remaining args, running them in order. -for script_arg in "${@:2}"; do - case ${script_arg} in - "--test") - test_containers - ;; - "--publish") - echo "Publishing..." - publish_containers - ;; - *) - echo "Error: Unrecognized argument '${script_arg}'" - break - ;; - esac -done diff --git a/scripts/test-containers.sh b/scripts/test-containers.sh new file mode 100755 index 000000000..517035677 --- /dev/null +++ b/scripts/test-containers.sh @@ -0,0 +1,60 @@ +# Clean up the CLI version for dev builds, since they aren't valid tag names. +# #!/bin/bash +# +# Builds the Pulumi docker containers locally. Optionally running tests or +# publishing to a container registry. +# +# Usage: build-docker cli-version [--test] [--publish] +set -o nounset +set -o errexit +set -o pipefail + +readonly SCRIPT_DIR="$( cd "$( dirname "${0}" )" && pwd )" +readonly ROOT=${SCRIPT_DIR}/.. + +if [ -z "${1:-}" ]; then + >&2 echo "error: missing version to publish" + exit 1 +fi + +# Sanitize the name of the version, e.g. +# "v1.14.0-alpha.1586190504+gf4e9f7e2" -> "v1.14.0-alpha.1586190504". +readonly CLI_VERSION="$(echo "${1}" | sed 's/\+.*//g')" + +# The Docker containers built/tested/published from this repository. +readonly PULUMI_CONTAINERS=("pulumi" "actions") + +echo_header() { + echo -e "\n\033[0;35m${1}\033[0m" +} + +echo_header "Building local copy of Pulumi containers (${CLI_VERSION})" +for container in ${PULUMI_CONTAINERS[@]}; do + echo "- Building pulumi/${container}" + docker build --build-arg PULUMI_VERSION="${CLI_VERSION}" \ + -t "pulumi/${container}:${CLI_VERSION}" \ + -t "pulumi/${container}:latest" \ + "${SCRIPT_DIR}/../docker/${container}" +done +echo_header "Executing container runtime tests" + +# Run the container tests, note that we also build the binaries into /tmp for the next step. +pushd ${ROOT}/tests +GOOS=linux go test -c -o /tmp/pulumi-test-containers ./containers/... +popd + +# Run tests _within_ the "pulumi" container, ensuring that the CLI is installed +# and working correctly. +docker run -e RUN_CONTAINER_TESTS=true \ + -e PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN} \ + --volume /tmp:/src \ + --entrypoint /bin/bash \ + pulumi/pulumi:latest \ + -c "pip install pipenv && /src/pulumi-test-containers -test.parallel=1 -test.timeout=1h -test.v -test.run TestPulumiDockerImage" + +# The actions container should fetch program dependencies from NPM, PIP, etc. before +# executing. These tests just shell out to docker run to confirm that. +echo_header "Executing container entrypoint tests" +pushd ${ROOT}/tests/containers +RUN_CONTAINER_TESTS=true go test . -test.run TestPulumiActionsImage -test.v -test.timeout=1h +popd diff --git a/scripts/update_homebrew.sh b/scripts/update_homebrew.sh deleted file mode 100755 index 82c4cf3cf..000000000 --- a/scripts/update_homebrew.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# update_homebrew.sh uses `brew bump-formula-pr` to update the formula for the Pulumi CLI and SDKs -set -o nounset -set -o errexit -set -o pipefail -readonly ROOT=$(dirname "${0}")/.. - -if [[ "${TRAVIS:-}" != "true" ]]; then - echo "error: this script should be run from within Travis" - exit 1 -fi - -if [[ -z "${PULUMI_BOT_GITHUB_API_TOKEN:-}" ]]; then - echo "error: PULUMI_BOT_GITHUB_API_TOKEN must be set" - exit 1 -fi - -if ! echo "${TRAVIS_TAG:-}" | grep -q -e "^v[0-9]\+\.[0-9]\+\.[0-9]\+$"; then - echo "Skipping Homebrew formula update; ${TRAVIS_TAG:-} does not denote a released version" - exit 0 -fi - -if [[ "${TRAVIS_OS_NAME:-}" != "osx" ]]; then - echo "Skipping Homebrew formula updte; not running on OS X" - exit 0 -fi - -HOMEBREW_GITHUB_API_TOKEN="${PULUMI_BOT_GITHUB_API_TOKEN:-}" brew bump-formula-pr --tag="${TRAVIS_TAG:-}" --revision="${TRAVIS_COMMIT:-}" pulumi -exit 0