Install plugin dependencies (#5353)

When installing a plugin, if it contains a `PulumiPlugin.yaml` file with a `runtime` value of `nodejs` or `python`, install dependencies for the plugin.

For Node.js, `npm install` is run (or `yarn install` if `PULUMI_PREFER_YARN` is set).

For Python, a virtual environment is created and deps installed into it.

pkg/backend/httpstate/policypack.go

@@ -15,7 +15,6 @@ import (
 	"github.com/pulumi/pulumi/pkg/v2/backend"
 	"github.com/pulumi/pulumi/pkg/v2/backend/httpstate/client"
 	"github.com/pulumi/pulumi/pkg/v2/engine"
-	"github.com/pulumi/pulumi/pkg/v2/npm"
 	resourceanalyzer "github.com/pulumi/pulumi/pkg/v2/resource/analyzer"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/tokens"
@@ -24,6 +23,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/result"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/workspace"
+	"github.com/pulumi/pulumi/sdk/v2/nodejs/npm"
 	"github.com/pulumi/pulumi/sdk/v2/python"
 )
 

pkg/cmd/pulumi/new.go

@@ -36,7 +36,6 @@ import (
 	"github.com/pulumi/pulumi/pkg/v2/backend/httpstate"
 	"github.com/pulumi/pulumi/pkg/v2/backend/state"
 	"github.com/pulumi/pulumi/pkg/v2/engine"
-	"github.com/pulumi/pulumi/pkg/v2/npm"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/diag/colors"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/resource/config"
@@ -46,6 +45,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/executable"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/workspace"
+	"github.com/pulumi/pulumi/sdk/v2/nodejs/npm"
 	"github.com/pulumi/pulumi/sdk/v2/python"
 )
 

scripts/go.sum

@@ -125,6 +125,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

sdk/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94
+	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cast v1.3.1
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.5.1

sdk/go.sum

@@ -153,6 +153,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0=
 github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

sdk/go/common/workspace/plugins.go

@@ -41,6 +41,8 @@ import (
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/httputil"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/version"
+	"github.com/pulumi/pulumi/sdk/v2/nodejs/npm"
+	"github.com/pulumi/pulumi/sdk/v2/python"
 )
 
 const (
@@ -228,7 +230,10 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error {
 	if err != nil {
 		return err
 	}
+	return installPlugin(finalDir, tarball)
+}
 
+func installPlugin(finalDir string, tarball io.ReadCloser) error {
 	// If part of the directory tree is missing, ioutil.TempDir will return an error, so make sure the path we're going
 	// to create the temporary folder in actually exists.
 	if err := os.MkdirAll(filepath.Dir(finalDir), 0700); err != nil {
@@ -261,6 +266,29 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error {
 		return err
 	}
 
+	// Install dependencies, if needed.
+	proj, err := LoadPluginProject(filepath.Join(tempDir, "PulumiPlugin.yaml"))
+	if err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "loading PulumiPlugin.yaml")
+	}
+	if proj != nil {
+		runtime := strings.ToLower(proj.Runtime.Name())
+		// For now, we only do this for Node.js and Python. For Go, the expectation is the binary is
+		// already built. For .NET, similarly, a single self-contained binary could be used, but
+		// otherwise `dotnet run` will implicitly run `dotnet restore`.
+		// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
+		switch runtime {
+		case "nodejs":
+			if _, err := npm.Install(tempDir, nil, os.Stderr); err != nil {
+				return errors.Wrap(err, "installing plugin dependencies")
+			}
+		case "python":
+			if err := python.InstallDependencies(tempDir, false /*showOutput*/, nil /*saveProj*/); err != nil {
+				return errors.Wrap(err, "installing plugin dependencies")
+			}
+		}
+	}
+
 	// If two calls to `plugin install` for the same plugin are racing, the second one will be unable to rename
 	// the directory. That's OK, just ignore the error. The temp directory created as part of the install will be
 	// cleaned up when we exit by the defer above.

sdk/go/common/workspace/plugins_install_nodejs_test.go

@@ -0,0 +1,36 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build nodejs all
+
+package workspace
+
+import (
+	"os"
+	"testing"
+)
+
+var tarball = map[string][]byte{
+	"PulumiPlugin.yaml": []byte("runtime: nodejs\n"),
+	"package.json":      []byte(`{"name":"test","dependencies":{"@pulumi/pulumi":"^2.0.0"}}`),
+}
+
+func TestNodeNPMInstall(t *testing.T) {
+	testPluginInstall(t, "node_modules", tarball)
+}
+
+func TestNodeYarnInstall(t *testing.T) {
+	os.Setenv("PULUMI_PREFER_YARN", "true")
+	testPluginInstall(t, "node_modules", tarball)
+}

sdk/go/common/workspace/plugins_install_python_test.go

@@ -0,0 +1,28 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build python all
+
+package workspace
+
+import (
+	"testing"
+)
+
+func TestPythonInstall(t *testing.T) {
+	testPluginInstall(t, "venv", map[string][]byte{
+		"PulumiPlugin.yaml": []byte("runtime: python\n"),
+		"package.json":      []byte("pulumi==2.0.0\n"),
+	})
+}

sdk/go/common/workspace/plugins_install_test.go

@@ -0,0 +1,105 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build nodejs python all
+
+package workspace
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// createTGZ creates an in-memory tarball.
+func createTGZ(files map[string][]byte) ([]byte, error) {
+	buffer := &bytes.Buffer{}
+	gw := gzip.NewWriter(buffer)
+	writer := tar.NewWriter(gw)
+
+	for name, content := range files {
+		if err := writer.WriteHeader(&tar.Header{
+			Name: name,
+			Size: int64(len(content)),
+			Mode: 0600,
+		}); err != nil {
+			return nil, err
+		}
+		if _, err := writer.Write(content); err != nil {
+			return nil, err
+		}
+	}
+
+	// Close the tar and gzip writers to flush and write footers.
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+	if err := gw.Close(); err != nil {
+		return nil, err
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func testPluginInstall(t *testing.T, expectedDir string, files map[string][]byte) {
+	// Skip during short test runs since this test involves downloading dependencies.
+	if testing.Short() {
+		t.Skip("Skipped in short test run")
+	}
+
+	tgz, err := createTGZ(files)
+	assert.NoError(t, err)
+
+	finalDirRoot, err := ioutil.TempDir("", "final-dir")
+	assert.NoError(t, err)
+	defer os.RemoveAll(finalDirRoot)
+	finalDir := filepath.Join(finalDirRoot, "final")
+
+	err = installPlugin(finalDir, ioutil.NopCloser(bytes.NewReader(tgz)))
+	assert.NoError(t, err)
+
+	info, err := os.Stat(filepath.Join(finalDir, expectedDir))
+	assert.NoError(t, err)
+	assert.True(t, info.IsDir())
+}
+
+func TestInstallNoDeps(t *testing.T) {
+	name := "foo.txt"
+	content := []byte("hello\n")
+
+	tgz, err := createTGZ(map[string][]byte{name: content})
+	assert.NoError(t, err)
+
+	finalDirRoot, err := ioutil.TempDir("", "final-dir")
+	assert.NoError(t, err)
+	defer os.RemoveAll(finalDirRoot)
+	finalDir := filepath.Join(finalDirRoot, "final")
+
+	err = installPlugin(finalDir, ioutil.NopCloser(bytes.NewReader(tgz)))
+	assert.NoError(t, err)
+
+	info, err := os.Stat(filepath.Join(finalDir, name))
+	assert.NoError(t, err)
+	assert.False(t, info.IsDir())
+
+	b, err := ioutil.ReadFile(filepath.Join(finalDir, name))
+	assert.NoError(t, err)
+	assert.Equal(t, content, b)
+}

sdk/go/common/workspace/project.go

@@ -155,6 +155,19 @@ func (proj *PolicyPackProject) Save(path string) error {
 	return save(path, proj, false /*mkDirAll*/)
 }
 
+type PluginProject struct {
+	// Runtime is a required runtime that executes code.
+	Runtime ProjectRuntimeInfo `json:"runtime" yaml:"runtime"`
+}
+
+func (proj *PluginProject) Validate() error {
+	if proj.Runtime.Name() == "" {
+		return errors.New("project is missing a 'runtime' attribute")
+	}
+
+	return nil
+}
+
 // ProjectStack holds stack specific information about a project.
 type ProjectStack struct {
 	// SecretsProvider is this stack's secrets provider.
@@ -319,6 +332,34 @@ func LoadPolicyPack(path string) (*PolicyPackProject, error) {
 	return &proj, err
 }
 
+// LoadPluginProject reads a plugin project definition from a file.
+func LoadPluginProject(path string) (*PluginProject, error) {
+	contract.Require(path != "", "path")
+
+	m, err := marshallerForPath(path)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var proj PluginProject
+	err = m.Unmarshal(b, &proj)
+	if err != nil {
+		return nil, err
+	}
+
+	err = proj.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	return &proj, err
+}
+
 // LoadProjectStack reads a stack definition from a file.
 func LoadProjectStack(path string) (*ProjectStack, error) {
 	contract.Require(path != "", "path")

sdk/nodejs/npm/npm.go

@@ -1,3 +1,17 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package npm
 
 import (

sdk/python/python.go

@@ -159,8 +159,10 @@ func InstallDependencies(root string, showOutput bool, saveProj func(virtualenv
 	}
 
 	// Save project with venv info.
-	if err := saveProj("venv"); err != nil {
-		return err
+	if saveProj != nil {
+		if err := saveProj("venv"); err != nil {
+			return err
+		}
 	}
 
 	print("Finished creating virtual environment")