WIP: compose

.travis.yml

@@ -1,5 +1,5 @@
 # It may be tempting to add parens around each individual clause in this expression, but Travis then builds pushes anyway
-if: branch = master OR branch =~ ^features/ OR branch =~ ^release/ OR tag IS present
+if: branch = master OR branch =~ ^release/ OR tag IS present OR type = api
 matrix:
   include:
     - os: linux
@@ -12,17 +12,19 @@ language: go
 go: 1.9
 sudo: true # give us 7.5GB and >2 bursted cores.
 git:
-  depth: false
+    depth: false
 before_install:
-    - source ./build/travis/prepare-environment.sh
-    - if [ "${TRAVIS_SECURE_ENV_VARS:-}" = "true" ]; then source "${PULUMI_SCRIPTS}/ci/keep-failed-tests.sh"; fi
+    - git clone https://github.com/pulumi/scripts ${GOPATH}/src/github.com/pulumi/scripts
+    - git -C ${GOPATH}/src/github.com/pulumi/scripts checkout origin/feature/ellismg/compose-build
+    - source ${GOPATH}/src/github.com/pulumi/scripts/ci/prepare-environment.sh
+    - source ${PULUMI_SCRIPTS}/ci/keep-failed-tests.sh
 install:
-    - source ./build/travis/install-common-toolchain.sh
+    - source ${PULUMI_SCRIPTS}/ci/install-common-toolchain.sh
 before_script:
-    - ./build/travis/ensure-dependencies
+    - ${PULUMI_SCRIPTS}/ci/ensure-dependencies
 script:
     - make travis_${TRAVIS_EVENT_TYPE}
 after_failure:
-    - if [ "${TRAVIS_SECURE_ENV_VARS:-}" = "true" ]; then "${PULUMI_SCRIPTS}/ci/upload-failed-tests"; fi
+    - ${PULUMI_SCRIPTS}/ci/upload-failed-tests
 notifications:
     webhooks: https://ufci1w66n3.execute-api.us-west-2.amazonaws.com/stage/travis

build/travis/ensure-dependencies

@@ -1,12 +0,0 @@
-#!/bin/bash
-set -o nounset -o errexit -o pipefail
-
-# Run our target to dep ensure and yarn install everything
-make ensure
-
-# Ensure the working tree is clean (make ensure may have updated lock files)
-$(go env GOPATH)/src/github.com/pulumi/scripts/ci/check-worktree-is-clean.sh
-
-# Set stdout back to blocking to avoid problems writing large outputs.
-# https://github.com/pulumi/pulumi-ppc/issues/176
-python -c 'import fcntl, os, sys; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); print("stdout was " + ("nonblocking" if flags & os.O_NONBLOCK else "blocking")); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)'

build/travis/install-common-toolchain.sh

@@ -1,96 +0,0 @@
-nvm install ${NODE_VERSION-v8.11.1}
-
-# Travis sources this script, so we can export variables into the
-# outer shell, so we don't want to set options like nounset because
-# they would be set in the outer shell as well, so do as much logic as
-# we can in a subshell.
-(
-    set -o nounset -o errexit -o pipefail
-    [ -e "$(go env GOPATH)/bin" ] || mkdir -p "$(go env GOPATH)/bin"
-
-    YARN_VERSION="1.3.2"
-    DEP_VERSION="0.4.1"
-    GOMETALINTER_VERSION="2.0.3"
-    PIP_VERSION="10.0.0"
-    VIRTUALENV_VERSION="15.2.0"
-    AWSCLI_VERSION="1.14.30"
-    WHEEL_VERSION="0.30.0"
-    TWINE_VERSION="1.9.1"
-
-    OS=""
-    case $(uname) in
-        "Linux") OS="linux";;
-        "Darwin") OS="darwin";;
-        *) echo "error: unknown host os $(uname)" ; exit 1;;
-    esac
-
-    # Tool installs and workarounds specific to macOS.
-    if [ "${TRAVIS_OS_NAME:-}" = "osx" ]; then
-        brew install jq
-        # On Travis, pip is called pip2.7, so alias it.
-        if [ ! -f /usr/local/bin/pip ]; then
-            sudo ln -s $(which pip2.7) /usr/local/bin/pip
-        fi
-    fi
-
-    echo "installing yarn ${YARN_VERSION}"
-    curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version ${YARN_VERSION}
-
-    echo "installing dep ${DEP_VERSION}"
-    curl -L -o "$(go env GOPATH)/bin/dep" https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-${OS}-amd64
-    chmod +x "$(go env GOPATH)/bin/dep"
-
-    echo "installing Gometalinter ${GOMETALINTER_VERSION}"
-    curl -L "https://github.com/alecthomas/gometalinter/releases/download/v${GOMETALINTER_VERSION}/gometalinter-v${GOMETALINTER_VERSION}-${OS}-amd64.tar.bz2" | tar -jxv --strip-components=1 -C "$(go env GOPATH)/bin"
-
-    chmod +x "$(go env GOPATH)/bin/gometalinter"
-    chmod +x "$(go env GOPATH)/bin/linters/"*
-
-    # Gometalinter looks for linters on the $PATH, so let's move them out
-    # of the linters folder and into GOBIN (which we know is on the $PATH)
-    mv "$(go env GOPATH)/bin/linters/"* "$(go env GOPATH)/bin/."
-    rm -rf "$(go env GOPATH)/bin/linters/"
-
-    echo "installing gocovmerge"
-
-    # gocovmerge does not publish versioned releases, but it also hasn't been updated in two years, so
-    # getting HEAD is pretty safe.
-    go get -v github.com/wadey/gocovmerge
-
-    echo "upgrading Pip to ${PIP_VERSION}"
-    sudo pip install --upgrade "pip>=${PIP_VERSION}"
-    pip install --user --upgrade "pip>=${PIP_VERSION}"
-
-    echo "installing virtualenv ${VIRTUALENV_VERSION}"
-    sudo pip install "virtualenv==${VIRTUALENV_VERSION}"
-    pip install --user "virtualenv==${VIRTUALENV_VERSION}"
-
-    echo "installing AWS cli ${AWSCLI_VERSION}"
-    pip install --user "awscli==${AWSCLI_VERSION}"
-
-    echo "installing Wheel and Twine, so we can publish Python packages"
-    pip install --user "wheel==${WHEEL_VERSION}" "twine==${TWINE_VERSION}"
-
-    echo "installing pandoc, so we can generate README.rst for Python packages"
-    if [ "${TRAVIS_OS_NAME:-}" = "linux" ]; then
-        sudo apt-get update
-        sudo apt-get install pandoc
-    else
-        brew install pandoc
-    fi
-)
-
-# If the sub shell failed, bail out now.
-[ "$?" -eq 0 ] || exit 1
-
-# By default some tools are not on the PATH, let's fix that
-
-# On OSX, the user folder that `pip` installs tools to is not on the
-# $PATH by default.
-if [ "${TRAVIS_OS_NAME:-}" = "osx" ]; then
-    export PATH=$PATH:$HOME/Library/Python/2.7/bin
-    export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python2.7/site-packages
-fi
-
-# Add yarn to the $PATH
-export PATH=$HOME/.yarn/bin:$PATH

build/travis/prepare-environment.sh

@@ -1,27 +0,0 @@
-# Travis sources this script, so we can export variables into the
-# outer shell, so we don't want to set options like nounset because
-# they would be set in the outer shell as well, so do as much logic as
-# we can in a subshell.
-
-export PULUMI_SCRIPTS="$(go env GOPATH)/src/github.com/pulumi/scripts"
-
-(
-    set -o nounset -o errexit -o pipefail
-
-    if [ "${TRAVIS_OS_NAME:-}" = "osx" ]; then
-        sudo mkdir /opt/pulumi
-        sudo chown "${USER}" /opt/pulumi
-    fi
-
-    git clone https://github.com/pulumi/scripts "${PULUMI_SCRIPTS}"
-
-    # If we have an NPM token, put it in the .npmrc file, so we can use it:
-    if [ ! -z "${NPM_TOKEN:-}" ]; then
-        echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" > ~/.npmrc
-    fi
-
-    # Put static entries for Pulumi backends in /etc/hosts
-    "${PULUMI_SCRIPTS}/ci/pulumi-hosts" | sudo tee -a /etc/hosts
-) || exit 1  # Abort outer script if subshell fails.
-
-export PULUMI_ROOT=/opt/pulumi

scripts/compose-skip-build

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# When doing a composed build, we only care about the leg that would publish packages.
+# We have additional legs that test other versions of node, which are un-interesting
+# in a composed build.
+[ "${TRAVIS_PUBLISH_PACKAGES}" != "true" ]

scripts/publish-compose-artifacts

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -eou pipefail
+source "${PULUMI_SCRIPTS}/compose/util.sh"
+
+if [ -z "${1:-}" ]; then
+    echo "usage: $0 <s3-prefix>"
+    exit 1
+fi
+
+S3_ROOT="$1"
+
+compose-publish-nodejs-package "./sdk/nodejs/bin" "${S3_ROOT}"
+compose-publish-binary "${PULUMI_ROOT}/bin/pulumi" "${S3_ROOT}"
+
+for resource in "${PULUMI_ROOT}/bin/"pulumi-resource-pulumi-*; do
+    compose-publish-binary "${resource}" "${S3_ROOT}"
+done
+
+for language in "${PULUMI_ROOT}/bin/"pulumi-language-*; do
+    compose-publish-binary "${language}" "${S3_ROOT}"
+done