Make virtualenv paths relative to root when main points elsewhere (#6966)

* Propagate workspace.Project metadata to plugin init

* Get to a working fix

* Propagate Root via plugin context

* Propagate root instead of yaml path

* Revert out unnecessary parameter propagation

* Root is now always absolute at this point; simplify code and docs

* Drop python conditional and propagate unused -root to all lang hosts

* Add tests that fail before and pass after

* Lint

* Add changelog entry

CHANGELOG_PENDING.md

@@ -1,7 +1,5 @@
 ### Breaking Changes
 
-
-
 ### Improvements
 
 - [auto/dotnet] - Provide PulumiFn implementation that allows runtime stack type
@@ -12,6 +10,9 @@
 
 ### Bug Fixes
 
+- [sdk/python] Fix relative `runtime:options:virtualenv` path resolution to ignore `main` project attribute
+  [#6966](https://github.com/pulumi/pulumi/pull/6966)
+
 - [auto/dotnet] - Disable Language Server Host logging and checking appsettings.json config
   [#7023](https://github.com/pulumi/pulumi/pull/7023)
 

pkg/cmd/pulumi/policy_publish.go

@@ -80,7 +80,7 @@ func newPolicyPublishCmd() *cobra.Command {
 				return err
 			}
 
-			plugctx, err := plugin.NewContext(cmdutil.Diag(), cmdutil.Diag(), nil, nil, pwd,
+			plugctx, err := plugin.NewContextWithRoot(cmdutil.Diag(), cmdutil.Diag(), nil, nil, pwd, projinfo.Root,
 				projinfo.Proj.Runtime.Options(), false, nil)
 			if err != nil {
 				return err

pkg/engine/deployment.go

@@ -47,7 +47,7 @@ func ProjectInfoContext(projinfo *Projinfo, host plugin.Host, config plugin.Conf
 	}
 
 	// Create a context for plugins.
-	ctx, err := plugin.NewContext(diag, statusDiag, host, config, pwd,
+	ctx, err := plugin.NewContextWithRoot(diag, statusDiag, host, config, pwd, projinfo.Root,
 		projinfo.Proj.Runtime.Options(), disableProviderPreview, tracingSpan)
 	if err != nil {
 		return "", "", nil, err

sdk/dotnet/cmd/pulumi-language-dotnet/main.go

@@ -47,8 +47,10 @@ var (
 func main() {
 	var tracing string
 	var binary string
+	var root string
 	flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.StringVar(&binary, "binary", "", "A relative or an absolute path to a precompiled .NET assembly to execute")
+	flag.StringVar(&root, "root", "", "Project root path to use")
 
 	// You can use the below flag to request that the language host load a specific executor instead of probing the
 	// PATH.  This can be used during testing to override the default location.

sdk/go/common/resource/plugin/context.go

@@ -25,23 +25,36 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
 )
 
-// Context is used to group related operations together so that associated OS resources can be cached, shared, and
-// reclaimed as appropriate.
+// Context is used to group related operations together so that
+// associated OS resources can be cached, shared, and reclaimed as
+// appropriate. It also carries shared plugin configuration.
 type Context struct {
 	Diag       diag.Sink // the diagnostics sink to use for messages.
 	StatusDiag diag.Sink // the diagnostics sink to use for status messages.
 	Host       Host      // the host that can be used to fetch providers.
 	Pwd        string    // the working directory to spawn all plugins in.
+	Root       string    // the root directory of the project.
 
 	tracingSpan opentracing.Span // the OpenTracing span to parent requests within.
 }
 
-// NewContext allocates a new context with a given sink and host.  Note that the host is "owned" by this context from
-// here forwards, such that when the context's resources are reclaimed, so too are the host's.
+// NewContext allocates a new context with a given sink and host. Note
+// that the host is "owned" by this context from here forwards, such
+// that when the context's resources are reclaimed, so too are the
+// host's.
 func NewContext(d, statusD diag.Sink, host Host, cfg ConfigSource,
 	pwd string, runtimeOptions map[string]interface{}, disableProviderPreview bool,
 	parentSpan opentracing.Span) (*Context, error) {
 
+	root := ""
+	return NewContextWithRoot(d, statusD, host, cfg, pwd, root, runtimeOptions, disableProviderPreview, parentSpan)
+}
+
+// Variation of NewContext that also sets known project Root.
+func NewContextWithRoot(d, statusD diag.Sink, host Host, cfg ConfigSource,
+	pwd, root string, runtimeOptions map[string]interface{}, disableProviderPreview bool,
+	parentSpan opentracing.Span) (*Context, error) {
+
 	if d == nil {
 		d = diag.DefaultSink(ioutil.Discard, ioutil.Discard, diag.FormatOptions{Color: colors.Never})
 	}

sdk/go/common/resource/plugin/langruntime_plugin.go

@@ -16,6 +16,7 @@ package plugin
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/blang/semver"
@@ -59,6 +60,13 @@ func NewLanguageRuntime(host Host, ctx *Context, runtime string,
 	for k, v := range options {
 		args = append(args, fmt.Sprintf("-%s=%v", k, v))
 	}
+
+	root, err := filepath.Abs(ctx.Root)
+	if err != nil {
+		return nil, err
+	}
+	args = append(args, fmt.Sprintf("-root=%s", filepath.Clean(root)))
+
 	args = append(args, host.ServerAddr())
 
 	plug, err := newPlugin(ctx, ctx.Pwd, path, runtime, args, nil /*env*/)

sdk/go/pulumi-language-go/main.go

@@ -79,8 +79,10 @@ func findProgram(binary string) (*exec.Cmd, error) {
 func main() {
 	var tracing string
 	var binary string
+	var root string
 	flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.StringVar(&binary, "binary", "", "Look on path for a binary executable with this name")
+	flag.StringVar(&root, "root", "", "Project root path to use")
 
 	flag.Parse()
 	args := flag.Args()

sdk/nodejs/cmd/pulumi-language-nodejs/main.go

@@ -77,10 +77,12 @@ const (
 func main() {
 	var tracing string
 	var typescript bool
+	var root string
 	flag.StringVar(&tracing, "tracing", "",
 		"Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.BoolVar(&typescript, "typescript", true,
 		"Use ts-node at runtime to support typescript source natively")
+	flag.StringVar(&root, "root", "", "Project root path to use")
 	flag.Parse()
 
 	args := flag.Args()

sdk/python/cmd/pulumi-language-python/main.go

@@ -63,8 +63,15 @@ const (
 func main() {
 	var tracing string
 	var virtualenv string
+	var root string
 	flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.StringVar(&virtualenv, "virtualenv", "", "Virtual environment path to use")
+	flag.StringVar(&root, "root", "", "Project root path to use")
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		cmdutil.Exit(errors.Wrapf(err, "getting the working directory"))
+	}
 
 	// You can use the below flag to request that the language host load a specific executor instead of probing the
 	// PATH.  This can be used during testing to override the default location.
@@ -105,10 +112,13 @@ func main() {
 		engineAddress = args[0]
 	}
 
+	// Resolve virtualenv path relative to root.
+	virtualenvPath := resolveVirtualEnvironmentPath(root, virtualenv)
+
 	// Fire up a gRPC server, letting the kernel choose a free port.
 	port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{
 		func(srv *grpc.Server) error {
-			host := newLanguageHost(pythonExec, engineAddress, tracing, virtualenv)
+			host := newLanguageHost(pythonExec, engineAddress, tracing, cwd, virtualenv, virtualenvPath)
 			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
 			return nil
 		},
@@ -132,15 +142,27 @@ type pythonLanguageHost struct {
 	exec          string
 	engineAddress string
 	tracing       string
-	virtualenv    string
+
+	// current working directory
+	cwd string
+
+	// virtualenv option as passed from Pulumi.yaml runtime.options.virtualenv.
+	virtualenv string
+
+	// if non-empty, points to the resolved directory path of the virtualenv
+	virtualenvPath string
 }
 
-func newLanguageHost(exec, engineAddress, tracing, virtualenv string) pulumirpc.LanguageRuntimeServer {
+func newLanguageHost(exec, engineAddress, tracing, cwd, virtualenv,
+	virtualenvPath string) pulumirpc.LanguageRuntimeServer {
+
 	return &pythonLanguageHost{
-		exec:          exec,
-		engineAddress: engineAddress,
-		tracing:       tracing,
-		virtualenv:    virtualenv,
+		cwd:            cwd,
+		exec:           exec,
+		engineAddress:  engineAddress,
+		tracing:        tracing,
+		virtualenv:     virtualenv,
+		virtualenvPath: virtualenvPath,
 	}
 }
 
@@ -148,19 +170,14 @@ func newLanguageHost(exec, engineAddress, tracing, virtualenv string) pulumirpc.
 func (host *pythonLanguageHost) GetRequiredPlugins(ctx context.Context,
 	req *pulumirpc.GetRequiredPluginsRequest) (*pulumirpc.GetRequiredPluginsResponse, error) {
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		return nil, errors.Wrap(err, "getting the working directory")
-	}
-
 	// Prepare the virtual environment (if needed).
-	virtualenv, err := host.prepareVirtualEnvironment(ctx, cwd)
+	err := host.prepareVirtualEnvironment(ctx, host.cwd)
 	if err != nil {
 		return nil, err
 	}
 
 	// Now, determine which Pulumi packages are installed.
-	pulumiPackages, err := determinePulumiPackages(virtualenv, cwd)
+	pulumiPackages, err := determinePulumiPackages(host.virtualenvPath, host.cwd)
 	if err != nil {
 		return nil, err
 	}
@@ -168,7 +185,7 @@ func (host *pythonLanguageHost) GetRequiredPlugins(ctx context.Context,
 	plugins := []*pulumirpc.PluginDependency{}
 	for _, pkg := range pulumiPackages {
 
-		plugin, err := determinePluginDependency(virtualenv, cwd, pkg.Name, pkg.Version)
+		plugin, err := determinePluginDependency(host.virtualenvPath, host.cwd, pkg.Name, pkg.Version)
 		if err != nil {
 			return nil, err
 		}
@@ -181,18 +198,24 @@ func (host *pythonLanguageHost) GetRequiredPlugins(ctx context.Context,
 	return &pulumirpc.GetRequiredPluginsResponse{Plugins: plugins}, nil
 }
 
-// prepareVirtualEnvironment will create and install dependencies in the virtual environment if host.virtualenv is set.
-// The full path to the virtual environment is returned.
-func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, cwd string) (string, error) {
-	virtualenv := host.virtualenv
+func resolveVirtualEnvironmentPath(root, virtualenv string) string {
 	if virtualenv == "" {
-		return "", nil
+		return ""
+	}
+	if !filepath.IsAbs(virtualenv) {
+		return filepath.Join(root, virtualenv)
+	}
+	return virtualenv
+}
+
+// prepareVirtualEnvironment will create and install dependencies in the virtual environment if host.virtualenv is set.
+func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, cwd string) error {
+
+	if host.virtualenv == "" {
+		return nil
 	}
 
-	// Make sure it's an absolute path.
-	if !filepath.IsAbs(virtualenv) {
-		virtualenv = filepath.Join(cwd, virtualenv)
-	}
+	virtualenv := host.virtualenvPath
 
 	// If the virtual environment directory doesn't exist, create it.
 	var createVirtualEnv bool
@@ -201,18 +224,17 @@ func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, c
 		if os.IsNotExist(err) {
 			createVirtualEnv = true
 		} else {
-			return "", err
+			return err
 		}
 	} else if !info.IsDir() {
-		return "",
-			errors.Errorf("the 'virtualenv' option in Pulumi.yaml is set to %q but it is not a directory", virtualenv)
+		return errors.Errorf("the 'virtualenv' option in Pulumi.yaml is set to %q but it is not a directory", virtualenv)
 	}
 
 	// If the virtual environment directory exists, but is empty, it needs to be created.
 	if !createVirtualEnv {
 		empty, err := fsutil.IsDirEmpty(virtualenv)
 		if err != nil {
-			return "", err
+			return err
 		}
 		createVirtualEnv = empty
 	}
@@ -226,7 +248,7 @@ func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, c
 			rpcutil.GrpcChannelOptions(),
 		)
 		if err != nil {
-			return "", errors.Wrapf(err, "language host could not make connection to engine")
+			return errors.Wrapf(err, "language host could not make connection to engine")
 		}
 
 		// Make a client around that connection.
@@ -251,17 +273,16 @@ func (host *pythonLanguageHost) prepareVirtualEnvironment(ctx context.Context, c
 
 		if err := python.InstallDependenciesWithWriters(
 			cwd, virtualenv, true /*showOutput*/, infoWriter, errorWriter); err != nil {
-			return "", err
+			return err
 		}
 	}
 
 	// Ensure the specified virtual directory is a valid virtual environment.
 	if !python.IsVirtualEnv(virtualenv) {
-		return "", python.NewVirtualEnvError(host.virtualenv, virtualenv)
+		return python.NewVirtualEnvError(host.virtualenv, virtualenv)
 	}
 
-	// Return the full path to the virtual environment.
-	return virtualenv, nil
+	return nil
 }
 
 type logWriter struct {
@@ -518,14 +539,7 @@ func (host *pythonLanguageHost) Run(ctx context.Context, req *pulumirpc.RunReque
 	var cmd *exec.Cmd
 	var virtualenv string
 	if host.virtualenv != "" {
-		virtualenv = host.virtualenv
-		if !filepath.IsAbs(virtualenv) {
-			cwd, err := os.Getwd()
-			if err != nil {
-				return nil, errors.Wrap(err, "getting the working directory")
-			}
-			virtualenv = filepath.Join(cwd, virtualenv)
-		}
+		virtualenv = host.virtualenvPath
 		if !python.IsVirtualEnv(virtualenv) {
 			return nil, python.NewVirtualEnvError(host.virtualenv, virtualenv)
 		}

tests/integration/integration_python_test.go

@@ -561,7 +561,7 @@ func TestAutomaticVenvCreation(t *testing.T) {
 	// handling by test harness; we actually are testing venv
 	// handling by the pulumi CLI itself.
 
-	check := func(t *testing.T, venvPathTemplate string) {
+	check := func(t *testing.T, venvPathTemplate string, dir string) {
 
 		e := ptesting.NewEnvironment(t)
 		defer func() {
@@ -573,7 +573,7 @@ func TestAutomaticVenvCreation(t *testing.T) {
 		venvPath := strings.ReplaceAll(venvPathTemplate, "${root}", e.RootPath)
 		t.Logf("venvPath = %s (IsAbs = %v)", venvPath, filepath.IsAbs(venvPath))
 
-		e.ImportDirectory(filepath.Join("python", "venv"))
+		e.ImportDirectory(dir)
 
 		// replace "virtualenv: venv" with "virtualenv: ${venvPath}" in Pulumi.yaml
 		pulumiYaml := filepath.Join(e.RootPath, "Pulumi.yaml")
@@ -586,6 +586,7 @@ func TestAutomaticVenvCreation(t *testing.T) {
 		newYaml := []byte(strings.ReplaceAll(string(oldYaml),
 			"virtualenv: venv",
 			fmt.Sprintf("virtualenv: >-\n      %s", venvPath)))
+
 		if err := ioutil.WriteFile(pulumiYaml, newYaml, 0644); err != nil {
 			t.Error(err)
 			return
@@ -611,11 +612,19 @@ func TestAutomaticVenvCreation(t *testing.T) {
 	}
 
 	t.Run("RelativePath", func(t *testing.T) {
-		check(t, "venv")
+		check(t, "venv", filepath.Join("python", "venv"))
 	})
 
 	t.Run("AbsolutePath", func(t *testing.T) {
-		check(t, filepath.Join("${root}", "absvenv"))
+		check(t, filepath.Join("${root}", "absvenv"), filepath.Join("python", "venv"))
+	})
+
+	t.Run("RelativePathWithMain", func(t *testing.T) {
+		check(t, "venv", filepath.Join("python", "venv-with-main"))
+	})
+
+	t.Run("AbsolutePathWithMain", func(t *testing.T) {
+		check(t, filepath.Join("${root}", "absvenv"), filepath.Join("python", "venv-with-main"))
 	})
 }
 

tests/integration/python/venv-with-main/.gitignore

@@ -0,0 +1,5 @@
+*.pyc
+/.pulumi/
+/dist/
+/*.egg-info
+venv/

tests/integration/python/venv-with-main/Pulumi.yaml

@@ -0,0 +1,7 @@
+name: pulumi-python-venv
+description: A simple Python Pulumi program that needs a venv to run.
+runtime:
+  name: python
+  options:
+    virtualenv: venv
+main: infra

tests/integration/python/venv-with-main/infra/__main__.py

@@ -0,0 +1,7 @@
+# Copyright 2016-2020, Pulumi Corporation.  All rights reserved.
+
+"""An example program that needs a venv to run"""
+
+import pulumi
+
+pulumi.export('foo', 'bar')

tests/integration/python/venv-with-main/infra/requirements.txt

@@ -0,0 +1 @@
+pulumi