From 162157c1a778a7dd4757eb54fc1581d6b35b79fc Mon Sep 17 00:00:00 2001 From: joeduffy Date: Sat, 29 Sep 2018 11:43:35 -0700 Subject: [PATCH] Add a Dockerfile for the Pulumi CLI This introduces a Dockerfile for the Pulumi CLI. This makes it easier to develop and test the engine in a self-contained environment, in addition to being suitable for running the actual CLI itself. For instance, $ docker run pulumi/pulumi -e "PULUMI_ACCESS_TOKEN=x" up will run the Pulumi program mounted under the /app volume. This will be used in some upcoming CI/CD scenarios. This uses multi-stage builds, and Debian Stretch as the base, for relatively fast and lean build times and resulting images. We are intentional about restoring dep packages independent of the actual source code so that we don't end up needlessly re-depping, which can consume quite a bit of time. After fixing https://github.com/pulumi/pulumi/issues/1986, we should explore an Alpine base image option. I made the decision to keep this image scoped to just the Go builds. Therefore, none of the actual SDK packages themselves are built, just the engine, CLI, and language plugins for Node.js, Python, and Go. It's possible to create a mega-container that has all of these full environments so that we can rebuild them too, but for now I figured it was better to rely on package management for them. Another alternative would have been to install released binaries, rather than building them. To keep the useful flow for development, however, I decided to go the build route for now. If we build at the same hashes, the resulting binaries "should" be ~identical anyhow. I've created a pulumi/pulumi Docker Hub repo that we can publish this into. For now, there is no CI publishing of the image. This fixes pulumi/pulumi#1991. --- .dockerignore | 4 ++++ Dockerfile | 43 +++++++++++++++++++++++++++++++++++++++++++ sdk/go/Makefile | 4 +++- sdk/nodejs/Makefile | 20 ++++++++++++++------ sdk/python/Makefile | 12 ++++++++++-- 5 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..a8df72773 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +examples/**/* +sdk/nodejs/bin/ +sdk/nodejs/node_modules/ +vendor/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d14554e9c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# Build the image in a distinct stage so we don't need the Golang SDK. +FROM golang:1.11-stretch as builder + +# Install pre-reqs. +# - Update apt-get sources +RUN apt-get update -y +# - Dep, for Go package management +RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + +# Copy the source code over, restore dependencies, and get ready to build everything. We copy the Gopkg +# files explicitly first to avoid excessive rebuild times when dependencies did not change. +WORKDIR /go/src/github.com/pulumi/pulumi +COPY Gopkg.* ./ +RUN dep ensure -v --vendor-only +COPY . . + +# Build the CLI itself. +RUN make install + +# Build the plguins for each of the language hosts (Go, Node.js, Python). This just builds Go code, and +# specifically avoids building the packages, since those are distributed using package managers. +RUN cd sdk/go && make install_plugin +RUN cd sdk/nodejs && make install_plugin +RUN cd sdk/python && make install_plugin + +# Install and run in Alpine Linux. +FROM debian:stretch + +# Copy over the binaries built during the prior stage. +COPY --from=builder /opt/pulumi/* /usr/local/bin/ + +# The app directory should contain the Pulumi program and is the pwd for the CLI. +WORKDIR /app +VOLUME ["/app"] + +# The app.pulumi.com access token is specified as an environment variable. You can create a new +# access token on your account page at https://app.pulumi.com/account. Please override this when +# running the Docker container using `docker run pulumi/pulumi -e "PULUMI_ACCESS_TOKEN=a1b2c2def9"`. +# ENV PULUMI_ACCESS_TOKEN + +# This image uses the `pulumi` CLI as an entrypoint. As a result, you may run commands simply by +# running `docker run pulumi/pulumi up` to run the program mounted in the `/app` volume location. +ENTRYPOINT ["pulumi", "--non-interactive"] diff --git a/sdk/go/Makefile b/sdk/go/Makefile index 53db62f1b..4abd61405 100644 --- a/sdk/go/Makefile +++ b/sdk/go/Makefile @@ -13,9 +13,11 @@ include ../../build/common.mk build:: go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGHOST_PKG} -install:: +install_plugin:: GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGHOST_PKG} +install:: install_plugin + lint:: $(GOMETALINTER) ./pulumi/... | sort $(GOMETALINTER) ./pulumi-language-go/... | sort diff --git a/sdk/nodejs/Makefile b/sdk/nodejs/Makefile index fbc64fef9..58264d4c1 100644 --- a/sdk/nodejs/Makefile +++ b/sdk/nodejs/Makefile @@ -17,11 +17,10 @@ export PATH:=$(shell yarn bin 2>/dev/null):$(PATH) lint:: $(GOMETALINTER) cmd/pulumi-language-nodejs/main.go | sort ; exit "$${PIPESTATUS[0]}" - tslint -c tslint.json -p tsconfig.json + ./node_modules/.bin/tslint -c tslint.json -p tsconfig.json -build:: - go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST} - tsc +build_package:: + ./node_modules/.bin/tsc cp tests/runtime/jsClosureCases.js bin/tests/runtime cp README.md ../../LICENSE package.json ./dist/* bin/ node ../../scripts/reversion.js bin/package.json ${VERSION} @@ -30,12 +29,21 @@ build:: mkdir -p bin/tests/runtime/langhost/cases/ find tests/runtime/langhost/cases/* -type d -exec cp -R {} bin/tests/runtime/langhost/cases/ \; -install:: - GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST} +build_plugin:: + go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST} + +build:: build_package build_plugin + +install_package:: cp dist/pulumi-resource-pulumi-nodejs "$(PULUMI_BIN)" mkdir -p "$(PULUMI_BIN)/v6.10.2" rm -rf "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)/tests" +install_plugin:: + GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST} + +install:: install_package install_plugin + test_fast:: istanbul test --print none _mocha -- --timeout 15000 'bin/tests/**/*.spec.js' istanbul report text-summary diff --git a/sdk/python/Makefile b/sdk/python/Makefile index 72f7a3a83..48b614f17 100644 --- a/sdk/python/Makefile +++ b/sdk/python/Makefile @@ -30,7 +30,7 @@ ensure:: pipenv $(PIPENV_PYTHON_VERSION) install --dev mkdir -p $(PYENVSRC) -build:: +build_package:: rm -rf $(PYENVSRC) && cp -R ./lib/. $(PYENVSRC)/ sed -i.bak "s/\$${VERSION}/$(VERSION)/g" $(PYENVSRC)/setup.py && rm $(PYENVSRC)/setup.py.bak if [ ! -z "$$(command -v pandoc)" ]; then \ @@ -40,19 +40,27 @@ build:: echo "" > "$(PYENVSRC)/README.rst"; \ fi cd $(PYENVSRC) && pipenv run python setup.py build bdist_wheel --universal + +build_plugin:: go install -ldflags "-X github.com/pulumi/pulumi/sdk/python/pkg/version.Version=${VERSION}" ${LANGHOST_PKG} +build:: build_package build_plugin + lint:: pipenv run pylint -E ./lib/pulumi --ignore-patterns '.*_pb2_.*.py' $(GOMETALINTER) ./cmd/pulumi-language-python/... | sort ; exit $${PIPESTATUS[0]} $(GOMETALINTER) ./pkg/... | sort ; exit $${PIPESTATUS[0]} -install:: +install_package:: cd $(PYENVSRC) && $(PIP) install --user -e . cp ./cmd/pulumi-language-python-exec "$(PULUMI_BIN)" + +install_plugin:: GOBIN=$(PULUMI_BIN) go install \ -ldflags "-X github.com/pulumi/pulumi/sdk/python/pkg/version.Version=${VERSION}" ${LANGHOST_PKG} +install:: install_package install_plugin + test_fast:: pipenv run pip install ./env/src pipenv run python -m unittest discover -s lib/test