diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 000000000..ea08dd854 --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,187 @@ +on: + push: + branches: + [ "master", "feature/**", "feature-**" ] +env: + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PULUMI_TEST_OWNER: "moolumi" + GO111MODULE: "on" + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + NUGET_PUBLISH_KEY: ${{ secrets.NUGET_PUBLISH_KEY }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TRAVIS_PUBLISH_PACKAGES: true + +jobs: + publish-sdks: + name: Publish SDKs + runs-on: ubuntu-latest + needs: publish-binaries + strategy: + matrix: + go-version: [ 1.14.x ] + python-version: [ 3.7.x ] + dotnet-version: [ 3.1.x ] + node-version: [ 10.x ] + language: [ "nodejs", "python", "dotnet" ] + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Set up DotNet ${{ matrix.dotnet-version }} + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet-version }} + - name: Set up Node ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org + always-auth: true + - name: Install pipenv + uses: dschep/install-pipenv-action@v1 + - name: Install Twine + run: python -m pip install pip twine + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Update path + run: | + echo "::add-path::${{ runner.temp }}/opt/pulumi/bin" + - name: Set Go Dep path + run: | + echo "::set-env name=PULUMI_GO_DEP_ROOT::$(dirname $(pwd))" + - name: Ensure + run: | + make ensure + - name: Publish Packages + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + make -C sdk/${{ matrix.language}} publish + publish-binaries: + name: Publish Binaries + runs-on: macos-latest + needs: build-and-test + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Install pulumictl + uses: jaxxstorm/action-install-gh-release@6277ebec57d2f9283d245d365f0b05bcc23d85e0 + with: + repo: pulumi/pulumictl + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: us-east-2 + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-external-id: upload-pulumi-release + role-session-name: pulumi@githubActions + role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE_ARN }} + - name: Set PreRelease Version + run: echo "::set-env name=GORELEASER_CURRENT_TAG::v$(pulumictl get version --language generic -o)" + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: -f .goreleaser.prerelease.yml --rm-dist --skip-validate + lint: + container: golangci/golangci-lint:latest + name: Lint ${{ matrix.directory }} + strategy: + matrix: + directory: [ sdk, pkg, tests ] + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Lint ${{ matrix.directory }} + run: | + cd ${{ matrix.directory }} && golangci-lint run -c ../.golangci.yml + build-and-test: + name: Build & Test + strategy: + matrix: + platform: [ ubuntu-latest, macos-latest ] + go-version: [1.14.x] + python-version: [ 3.7.x ] + dotnet-version: [ 3.1.x ] + node-version: [ 10.x ] + runs-on: ${{ matrix.platform }} + steps: + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Set up DotNet ${{ matrix.dotnet-version }} + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet-version }} + - name: Set up Node ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install pipenv + uses: dschep/install-pipenv-action@v1 + - name: Setup git + run: | + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + - name: Update path + run: | + echo "::add-path::${{ runner.temp }}/opt/pulumi/bin" + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Fetch Tags + run: | + git fetch --quiet --prune --unshallow --tags + - name: Set Go Dep path + run: | + echo "::set-env name=PULUMI_GO_DEP_ROOT::$(dirname $(pwd))" + - name: Ensure + run: | + make ensure + - name: Dist + run: | + make dist + env: + PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules + PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget + PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi + - name: Install + run: | + make install_all + env: + PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules + PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget + PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi + - name: Test + run: | + make test_all + env: + PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules + PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget + PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi + + diff --git a/.goreleaser.prerelease.yml b/.goreleaser.prerelease.yml new file mode 100644 index 000000000..0c0341375 --- /dev/null +++ b/.goreleaser.prerelease.yml @@ -0,0 +1,171 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +dist: goreleaser +project_name: pulumi +before: + hooks: + - cd sdk && go mod tidy + - cd sdk && go mod download + - cd pkg && go mod tidy + - cd pkg && go mod download +blobs: +- bucket: get.pulumi.com + folder: releases/sdk/ + ids: + - pulumi-unix + - pulumi-windows + provider: s3 + region: us-west-2 +changelog: + skip: true +release: + disable: true +builds: +# UNIX builds +- id: pulumi-unix + binary: pulumi + dir: pkg + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./cmd/pulumi +- id: pulumi-language-nodejs-unix + binary: pulumi-language-nodejs + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./nodejs/cmd/pulumi-language-nodejs +- id: pulumi-language-python-unix + binary: pulumi-language-python + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./python/cmd/pulumi-language-python +- id: pulumi-language-dotnet-unix + binary: pulumi-language-dotnet + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./dotnet/cmd/pulumi-language-dotnet +- id: pulumi-language-go-unix + binary: pulumi-language-go + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./go/pulumi-language-go + +# Windows builds +- id: pulumi-windows + binary: pulumi + dir: pkg + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./cmd/pulumi +- id: pulumi-language-nodejs-windows + binary: pulumi-language-nodejs + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./nodejs/cmd/pulumi-language-nodejs +- id: pulumi-language-python-windows + binary: pulumi-language-python + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./python/cmd/pulumi-language-python +- id: pulumi-language-dotnet-windows + binary: pulumi-language-dotnet + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./dotnet/cmd/pulumi-language-dotnet +- id: pulumi-language-go-windows + binary: pulumi-language-go + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./go/pulumi-language-go +archives: +- id: pulumi-unix + builds: + - pulumi-language-dotnet-unix + - pulumi-language-go-unix + - pulumi-language-python-unix + - pulumi-language-nodejs-unix + - pulumi-unix + replacements: + amd64: x64 + files: + - sdk/nodejs/dist/pulumi-resource-pulumi-nodejs + - sdk/python/dist/pulumi-resource-pulumi-python + - sdk/nodejs/dist/pulumi-analyzer-policy + - sdk/python/dist/pulumi-analyzer-policy-python + - sdk/python/cmd/pulumi-language-python-exec + name_template: "{{ .ProjectName }}-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" +- id: pulumi-windows + builds: + - pulumi-language-dotnet-windows + - pulumi-language-go-windows + - pulumi-language-python-windows + - pulumi-language-nodejs-windows + - pulumi-windows + replacements: + amd64: x64 + format_overrides: + - goos: windows + format: zip + files: + - sdk/nodejs/dist/pulumi-resource-pulumi-nodejs.cmd + - sdk/python/dist/pulumi-resource-pulumi-python.cmd + - sdk/nodejs/dist/pulumi-analyzer-policy.cmd + - sdk/python/dist/pulumi-analyzer-policy-python.cmd + - sdk/python/cmd/pulumi-language-python-exec + name_template: "{{ .ProjectName }}-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" +snapshot: + name_template: "{{ .Version }}-SNAPSHOT" +checksum: + name_template: "{{ .ProjectName }}-{{ .Version }}-checksums.txt" diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 000000000..78dbfc2c3 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,159 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +dist: goreleaser +project_name: pulumi +before: + hooks: + - cd sdk && go mod tidy + - cd sdk && go mod download + - cd pkg && go mod tidy + - cd pkg && go mod download +builds: +# UNIX builds +- id: pulumi-unix + binary: pulumi + dir: pkg + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./cmd/pulumi +- id: pulumi-language-nodejs-unix + binary: pulumi-language-nodejs + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./nodejs/cmd/pulumi-language-nodejs +- id: pulumi-language-python-unix + binary: pulumi-language-python + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./python/cmd/pulumi-language-python +- id: pulumi-language-dotnet-unix + binary: pulumi-language-dotnet + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./dotnet/cmd/pulumi-language-dotnet +- id: pulumi-language-go-unix + binary: pulumi-language-go + dir: sdk + goarch: + - amd64 + goos: + - linux + - darwin + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./go/pulumi-language-go + +# Windows builds +- id: pulumi-windows + binary: pulumi + dir: pkg + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./cmd/pulumi +- id: pulumi-language-nodejs-windows + binary: pulumi-language-nodejs + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./nodejs/cmd/pulumi-language-nodejs +- id: pulumi-language-python-windows + binary: pulumi-language-python + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./python/cmd/pulumi-language-python +- id: pulumi-language-dotnet-windows + binary: pulumi-language-dotnet + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./dotnet/cmd/pulumi-language-dotnet +- id: pulumi-language-go-windows + binary: pulumi-language-go + dir: sdk + goarch: + - amd64 + goos: + - windows + ldflags: + - github.com/pulumi/pulumi/pkg/v2/version.Version={{.Tag}} + main: ./go/pulumi-language-go +archives: +- id: pulumi-unix + builds: + - pulumi-language-dotnet-unix + - pulumi-language-go-unix + - pulumi-language-python-unix + - pulumi-language-nodejs-unix + - pulumi-unix + replacements: + amd64: x64 + files: + - sdk/nodejs/dist/pulumi-resource-pulumi-nodejs + - sdk/python/dist/pulumi-resource-pulumi-python + - sdk/nodejs/dist/pulumi-analyzer-policy + - sdk/python/dist/pulumi-analyzer-policy-python + - sdk/python/cmd/pulumi-language-python-exec + name_template: "{{ .ProjectName }}-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" +- id: pulumi-windows + builds: + - pulumi-language-dotnet-windows + - pulumi-language-go-windows + - pulumi-language-python-windows + - pulumi-language-nodejs-windows + - pulumi-windows + replacements: + amd64: x64 + format_overrides: + - goos: windows + format: zip + files: + - sdk/nodejs/dist/pulumi-resource-pulumi-nodejs.cmd + - sdk/python/dist/pulumi-resource-pulumi-python.cmd + - sdk/nodejs/dist/pulumi-analyzer-policy.cmd + - sdk/python/dist/pulumi-analyzer-policy-python.cmd + - sdk/python/cmd/pulumi-language-python-exec + name_template: "{{ .ProjectName }}-{{ .Tag }}-{{ .Os }}-{{ .Arch }}" +snapshot: + name_template: "{{ .Version }}-SNAPSHOT" +checksum: + name_template: "{{ .ProjectName }}-{{ .Version }}-checksums.txt" diff --git a/Makefile b/Makefile index 0b81a9de2..e23007d8f 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,9 @@ test_containers_cron: # The travis_* targets are entrypoints for CI. .PHONY: travis_cron travis_push travis_pull_request travis_api travis_cron: install dist all -travis_push: install dist publish_tgz only_test publish_packages +travis_push: + $(call STEP_MESSAGE) + @echo moved to GitHub Actions travis_pull_request: $(call STEP_MESSAGE) @echo moved to GitHub Actions diff --git a/scripts/publish_npm.sh b/scripts/publish_npm.sh new file mode 100755 index 000000000..59a5e1659 --- /dev/null +++ b/scripts/publish_npm.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# publish_npm.sh uploads our packages to npm +set -o nounset +set -o errexit +set -o pipefail +readonly ROOT=$(dirname "${0}")/.. + +if [[ "${TRAVIS_PUBLISH_PACKAGES:-}" == "true" ]]; then + echo "Publishing NPM package to NPMjs.com:" + NPM_TAG="dev" + + if [[ "${TRAVIS_BRANCH:-}" == features/* ]]; then + NPM_TAG=$(echo "${TRAVIS_BRANCH}" | sed -e 's|^features/|feature-|g') + fi + + if [[ "${TRAVIS_BRANCH:-}" == feature-* ]]; then + NPM_TAG=$(echo "${TRAVIS_BRANCH}") + fi + + PKG_NAME=$(jq -r .name < "${ROOT}/sdk/nodejs/bin/package.json") + PKG_VERSION=$(jq -r .version < "${ROOT}/sdk/nodejs/bin/package.json") + + # If the package doesn't have an alpha tag, use the tag of latest instead of + # dev. NPM uses this tag as the default version to add, so we want it to mean + # the newest released version. + if [[ "${PKG_VERSION}" != *-alpha* ]]; then + NPM_TAG="latest" + fi + + # Now, perform the publish. The logic here is a little goofy because npm provides + # no way to say "if the package already exists, don't fail" but we want these + # semantics (so, for example, we can restart builds which may have failed after + # publishing, or so two builds can run concurrently, which is the case for when we + # tag master right after pushing a new commit and the push and tag travis jobs both + # get the same version. + # + # We exploit the fact that `npm info @` has no output + # when the package does not exist. + pushd "${ROOT}/sdk/nodejs/bin" + if [ "$(npm info ${PKG_NAME}@${PKG_VERSION})" == "" ]; then + if ! npm publish -tag "${NPM_TAG}"; then + # if we get here, we have a TOCTOU issue, so check again + # to see if it published. If it didn't bail out. + if [ "$(npm info ${PKG_NAME}@${PKG_VERSION})" == "" ]; then + echo "NPM publishing failed, aborting" + exit 1 + fi + fi + fi + npm info 2>/dev/null + popd +fi diff --git a/sdk/dotnet/Makefile b/sdk/dotnet/Makefile index 0155db264..08388effa 100644 --- a/sdk/dotnet/Makefile +++ b/sdk/dotnet/Makefile @@ -61,3 +61,9 @@ dist:: brew:: dist go install -ldflags "-X github.com/pulumi/pulumi/sdk/v2/go/common/version.Version=${VERSION}" ${LANGHOST_PKG} + +publish:: build install + echo "Publishing .nupkgs to nuget.org:" + find /opt/pulumi/nuget -name 'Pulumi*.nupkg' \ + -exec dotnet nuget push -k ${NUGET_PUBLISH_KEY} -s https://api.nuget.org/v3/index.json {} ';' + diff --git a/sdk/nodejs/Makefile b/sdk/nodejs/Makefile index c8f163c4d..aaaea1a52 100644 --- a/sdk/nodejs/Makefile +++ b/sdk/nodejs/Makefile @@ -64,3 +64,6 @@ brew:: go install -ldflags "-X github.com/pulumi/pulumi/sdk/v2/go/common/version.Version=${VERSION}" ${LANGUAGE_HOST} cp dist/pulumi-resource-pulumi-nodejs "$$(go env GOPATH)"/bin/ cp dist/pulumi-analyzer-policy "$$(go env GOPATH)"/bin/ + +publish:: build_package + bash -c ../../scripts/publish_npm.sh diff --git a/sdk/python/Makefile b/sdk/python/Makefile index a6d705a4a..56ea34c3c 100644 --- a/sdk/python/Makefile +++ b/sdk/python/Makefile @@ -52,3 +52,10 @@ dist:: cp ./dist/pulumi-analyzer-policy-python "$$(go env GOPATH)"/bin/ brew:: dist + +publish:: build_package + twine upload \ + -u pulumi -p "${PYPI_PASSWORD}" \ + "env/src/dist"/*.whl \ + --skip-existing \ + --verbose