From b77ec919d4957cae450003bd8d28cf634a9703fe Mon Sep 17 00:00:00 2001 From: Justin Van Patten Date: Tue, 9 Jun 2020 23:42:53 +0000 Subject: [PATCH] Install and use dependencies automatically for new Python projects (#4775) Automatically create a virtual environment and install dependencies in it with `pulumi new` and `pulumi policy new` for Python templates. This will save a new `virtualenv` runtime option in `Pulumi.yaml` (`PulumiPolicy.yaml` for policy packs): ```yaml runtime: name: python options: virtualenv: venv ``` `virtualenv` is the path to a virtual environment that Pulumi will use when running `python` commands. Existing projects are unaffected and can opt-in to using this by setting `virtualenv`, otherwise, they'll continue to work as-is. --- CHANGELOG.md | 3 + pkg/backend/httpstate/policypack.go | 41 +--- pkg/cmd/pulumi/new.go | 33 ++- pkg/cmd/pulumi/policy_new.go | 42 ++-- pkg/cmd/pulumi/policy_publish.go | 2 +- pkg/cmd/pulumi/util.go | 12 +- pkg/testing/integration/program.go | 197 +++++++++++++++--- sdk/go/common/resource/plugin/host.go | 2 +- .../common/resource/plugin/provider_plugin.go | 12 +- sdk/python/cmd/pulumi-language-python/main.go | 44 +++- .../dist/pulumi-analyzer-policy-python.cmd | 14 +- sdk/python/dist/pulumi-resource-pulumi-python | 31 ++- .../dist/pulumi-resource-pulumi-python.cmd | 24 ++- sdk/python/python.go | 89 +++++++- sdk/python/python_test.go | 19 ++ .../config_basic/python_venv/.gitignore | 5 + .../config_basic/python_venv/Pulumi.yaml | 3 + .../config_basic/python_venv/__main__.py | 53 +++++ .../config_basic/python_venv/requirements.txt | 0 .../dynamic/python_venv/.gitignore | 5 + .../dynamic/python_venv/Pulumi.yaml | 3 + .../dynamic/python_venv/__main__.py | 21 ++ .../dynamic/python_venv/requirements.txt | 0 .../dynamic/python_venv/step1/README.md | 1 + .../integration/empty/python_venv/.gitignore | 5 + .../integration/empty/python_venv/Pulumi.yaml | 3 + .../integration/empty/python_venv/__main__.py | 7 + .../empty/python_venv/requirements.txt | 0 tests/integration/integration_test.go | 65 ++++++ 29 files changed, 608 insertions(+), 128 deletions(-) create mode 100644 tests/integration/config_basic/python_venv/.gitignore create mode 100644 tests/integration/config_basic/python_venv/Pulumi.yaml create mode 100644 tests/integration/config_basic/python_venv/__main__.py create mode 100644 tests/integration/config_basic/python_venv/requirements.txt create mode 100644 tests/integration/dynamic/python_venv/.gitignore create mode 100644 tests/integration/dynamic/python_venv/Pulumi.yaml create mode 100644 tests/integration/dynamic/python_venv/__main__.py create mode 100644 tests/integration/dynamic/python_venv/requirements.txt create mode 100644 tests/integration/dynamic/python_venv/step1/README.md create mode 100644 tests/integration/empty/python_venv/.gitignore create mode 100644 tests/integration/empty/python_venv/Pulumi.yaml create mode 100644 tests/integration/empty/python_venv/__main__.py create mode 100644 tests/integration/empty/python_venv/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 2538fddf8..7560f31c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ CHANGELOG - Allow users to specify base64 encoded strings as GOOGLE_CREDENTIALS [#4773](https://github.com/pulumi/pulumi/pull/4773) +- Install and use dependencies automatically for new Python projects. + [#4775](https://github.com/pulumi/pulumi/pull/4775) + --- ## 2.3.0 (2020-05-27) diff --git a/pkg/backend/httpstate/policypack.go b/pkg/backend/httpstate/policypack.go index 9cb8d1afd..250e3b773 100644 --- a/pkg/backend/httpstate/policypack.go +++ b/pkg/backend/httpstate/policypack.go @@ -320,42 +320,15 @@ func completeNodeJSInstall(finalDir string) error { } func completePythonInstall(finalDir, projPath string, proj *workspace.PolicyPackProject) error { - // Create virtual environment. - venvDir := filepath.Join(finalDir, "venv") - cmd, err := python.Command("-m", "venv", venvDir) - if err != nil { - return err - } - - if output, err := cmd.CombinedOutput(); err != nil { - if len(output) > 0 { - os.Stdout.Write(output) - fmt.Println() + if err := python.InstallDependencies(finalDir, false /*showOutput*/, func(virtualenv string) error { + // Save project with venv info. + proj.Runtime.SetOption("virtualenv", virtualenv) + if err := proj.Save(projPath); err != nil { + return errors.Wrapf(err, "saving project at %s", projPath) } - return errors.Wrapf(err, "creating virtual environment at %s", venvDir) - } - - // Save project with venv info. - proj.Runtime.SetOption("virtualenv", "venv") - if err := proj.Save(projPath); err != nil { - return errors.Wrapf(err, "saving project at %s", projPath) - } - - requirementsPath := filepath.Join(finalDir, "requirements.txt") - if _, err := os.Stat(requirementsPath); os.IsNotExist(err) { return nil - } - - pipCmd := python.VirtualEnvCommand(venvDir, "pip", "install", "-r", "requirements.txt") - pipCmd.Dir = finalDir - pipCmd.Env = python.ActivateVirtualEnv(os.Environ(), venvDir) - - if output, err := pipCmd.CombinedOutput(); err != nil { - if len(output) > 0 { - os.Stdout.Write(output) - fmt.Println() - } - return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`") + }); err != nil { + return err } fmt.Println("Finished installing policy pack") diff --git a/pkg/cmd/pulumi/new.go b/pkg/cmd/pulumi/new.go index a926f7b3c..f2ef37f3c 100644 --- a/pkg/cmd/pulumi/new.go +++ b/pkg/cmd/pulumi/new.go @@ -46,6 +46,7 @@ import ( "github.com/pulumi/pulumi/sdk/v2/go/common/util/executable" "github.com/pulumi/pulumi/sdk/v2/go/common/util/logging" "github.com/pulumi/pulumi/sdk/v2/go/common/workspace" + "github.com/pulumi/pulumi/sdk/v2/python" ) type promptForValueFunc func(yes bool, valueType string, defaultValue string, secret bool, @@ -568,6 +569,8 @@ func installDependencies(proj *workspace.Project, root string) error { return errors.Wrapf(err, "%s install failed; rerun manually to try again, "+ "then run 'pulumi up' to perform an initial deployment", bin) } + } else if strings.EqualFold(proj.Runtime.Name(), "python") { + return pythonInstallDependencies(proj, root) } else if strings.EqualFold(proj.Runtime.Name(), "dotnet") { return dotnetInstallDependenciesAndBuild(proj, root) } else if strings.EqualFold(proj.Runtime.Name(), "go") { @@ -597,6 +600,18 @@ func nodeInstallDependencies() (string, error) { return bin, nil } +// pythonInstallDependencies will create a new virtual environment and install dependencies. +func pythonInstallDependencies(proj *workspace.Project, root string) error { + return python.InstallDependencies(root, true /*showOutput*/, func(virtualenv string) error { + // Save project with venv info. + proj.Runtime.SetOption("virtualenv", virtualenv) + if err := workspace.SaveProject(proj); err != nil { + return errors.Wrap(err, "saving project") + } + return nil + }) +} + // dotnetInstallDependenciesAndBuild will install dependencies and build the project. func dotnetInstallDependenciesAndBuild(proj *workspace.Project, root string) error { contract.Assert(proj != nil) @@ -672,18 +687,14 @@ func printNextSteps(proj *workspace.Project, originalCwd, cwd string, generateOn commands = append(commands, cd) } - if strings.EqualFold(proj.Runtime.Name(), "nodejs") && generateOnly { - // If we're generating a NodeJS project, and we didn't install dependencies (generateOnly), - // instruct the user to do so. - commands = append(commands, "npm install") - } else if strings.EqualFold(proj.Runtime.Name(), "python") { - // If we're generating a Python project, instruct the user to set up and activate a virtual - // environment. - commands = append(commands, pythonCommands()...) - } - - // If we didn't create a stack, show that as a command to run before `pulumi up`. if generateOnly { + // We didn't install dependencies, so instruct the user to do so. + if strings.EqualFold(proj.Runtime.Name(), "nodejs") { + commands = append(commands, "npm install") + } else if strings.EqualFold(proj.Runtime.Name(), "python") { + commands = append(commands, pythonCommands()...) + } + // We didn't create a stack so show that as a command to run before `pulumi up`. commands = append(commands, "pulumi stack init") } diff --git a/pkg/cmd/pulumi/policy_new.go b/pkg/cmd/pulumi/policy_new.go index 506cdd8a0..3976276d1 100644 --- a/pkg/cmd/pulumi/policy_new.go +++ b/pkg/cmd/pulumi/policy_new.go @@ -26,6 +26,7 @@ import ( "github.com/pulumi/pulumi/sdk/v2/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v2/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v2/go/common/workspace" + "github.com/pulumi/pulumi/sdk/v2/python" "github.com/spf13/cobra" survey "gopkg.in/AlecAivazis/survey.v1" surveycore "gopkg.in/AlecAivazis/survey.v1/core" @@ -162,14 +163,14 @@ func runNewPolicyPack(args newPolicyArgs) error { fmt.Println("Created Policy Pack!") - proj, root, err := readPolicyProject() + proj, projPath, root, err := readPolicyProject() if err != nil { return err } // Install dependencies. if !args.generateOnly { - if err := installPolicyPackDependencies(proj); err != nil { + if err := installPolicyPackDependencies(proj, projPath, root); err != nil { return err } } @@ -185,26 +186,36 @@ func runNewPolicyPack(args newPolicyArgs) error { return nil } -func installPolicyPackDependencies(proj *workspace.PolicyPackProject) error { +func installPolicyPackDependencies(proj *workspace.PolicyPackProject, projPath, root string) error { // TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here. if strings.EqualFold(proj.Runtime.Name(), "nodejs") { if bin, err := nodeInstallDependencies(); err != nil { return errors.Wrapf(err, "`%s install` failed; rerun manually to try again.", bin) } + } else if strings.EqualFold(proj.Runtime.Name(), "python") { + if err := python.InstallDependencies(root, true /*showOutput*/, func(virtualenv string) error { + // Save project with venv info. + proj.Runtime.SetOption("virtualenv", virtualenv) + if err := proj.Save(projPath); err != nil { + return errors.Wrapf(err, "saving project at %s", projPath) + } + return nil + }); err != nil { + return err + } } return nil } func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, generateOnly bool, opts display.Options) { var commands []string - if strings.EqualFold(proj.Runtime.Name(), "nodejs") && generateOnly { - // If we're generating a NodeJS policy pack, and we didn't install dependencies - // (generateOnly), instruct the user to do so. - commands = append(commands, "npm install") - } else if strings.EqualFold(proj.Runtime.Name(), "python") { - // If we're generating a Python policy pack, instruct the user to set up and - // activate a virtual environment. - commands = append(commands, pythonCommands()...) + if generateOnly { + // We didn't install dependencies, so instruct the user to do so. + if strings.EqualFold(proj.Runtime.Name(), "nodejs") { + commands = append(commands, "npm install") + } else if strings.EqualFold(proj.Runtime.Name(), "python") { + commands = append(commands, pythonCommands()...) + } } if len(commands) == 1 { @@ -228,8 +239,7 @@ func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, ge []string{"run the Policy Pack against a Pulumi program, in the directory of the Pulumi program run"} usageCommands := []string{fmt.Sprintf("pulumi up --policy-pack %s", root)} - // Currently, only Node.js Policy Packs can be published. - if strings.EqualFold(proj.Runtime.Name(), "nodejs") { + if strings.EqualFold(proj.Runtime.Name(), "nodejs") || strings.EqualFold(proj.Runtime.Name(), "python") { usageCommandPreambles = append(usageCommandPreambles, "publish the Policy Pack, run") usageCommands = append(usageCommands, "pulumi policy publish [org-name]") } @@ -251,12 +261,6 @@ func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, ge } fmt.Println() } - - // Add special note for Python. - if strings.EqualFold(proj.Runtime.Name(), "python") { - fmt.Println("Note: When running the Policy Pack against a Pulumi program, if the Pulumi program is " + - "also Python, both the Pulumi program and Policy Pack must use the same virtual environment.") - } } // choosePolicyPackTemplate will prompt the user to choose amongst the available templates. diff --git a/pkg/cmd/pulumi/policy_publish.go b/pkg/cmd/pulumi/policy_publish.go index 47b56a88c..67daf8281 100644 --- a/pkg/cmd/pulumi/policy_publish.go +++ b/pkg/cmd/pulumi/policy_publish.go @@ -69,7 +69,7 @@ func newPolicyPublishCmd() *cobra.Command { // Load metadata about the current project. // - proj, root, err := readPolicyProject() + proj, _, root, err := readPolicyProject() if err != nil { return err } diff --git a/pkg/cmd/pulumi/util.go b/pkg/cmd/pulumi/util.go index 414f2426c..9a71b73e2 100644 --- a/pkg/cmd/pulumi/util.go +++ b/pkg/cmd/pulumi/util.go @@ -402,26 +402,26 @@ func readProject() (*workspace.Project, string, error) { // readPolicyProject attempts to detect and read a Pulumi PolicyPack project for the current // workspace. If the project is successfully detected and read, it is returned along with the path // to its containing directory, which will be used as the root of the project's Pulumi program. -func readPolicyProject() (*workspace.PolicyPackProject, string, error) { +func readPolicyProject() (*workspace.PolicyPackProject, string, string, error) { pwd, err := os.Getwd() if err != nil { - return nil, "", err + return nil, "", "", err } // Now that we got here, we have a path, so we will try to load it. path, err := workspace.DetectPolicyPackPathFrom(pwd) if err != nil { - return nil, "", errors.Wrapf(err, "failed to find current Pulumi project because of "+ + return nil, "", "", errors.Wrapf(err, "failed to find current Pulumi project because of "+ "an error when searching for the PulumiPolicy.yaml file (searching upwards from %s)", pwd) } else if path == "" { - return nil, "", fmt.Errorf("no PulumiPolicy.yaml project file found (searching upwards from %s)", pwd) + return nil, "", "", fmt.Errorf("no PulumiPolicy.yaml project file found (searching upwards from %s)", pwd) } proj, err := workspace.LoadPolicyPack(path) if err != nil { - return nil, "", errors.Wrapf(err, "failed to load Pulumi policy project located at %q", path) + return nil, "", "", errors.Wrapf(err, "failed to load Pulumi policy project located at %q", path) } - return proj, filepath.Dir(path), nil + return proj, path, filepath.Dir(path), nil } // anyWriter is an io.Writer that will set itself to `true` iff any call to `anyWriter.Write` is made with a diff --git a/pkg/testing/integration/program.go b/pkg/testing/integration/program.go index 822f71121..246c9a287 100644 --- a/pkg/testing/integration/program.go +++ b/pkg/testing/integration/program.go @@ -60,6 +60,8 @@ const NodeJSRuntime = "nodejs" const GoRuntime = "go" const DotNetRuntime = "dotnet" +const windowsOS = "windows" + // RuntimeValidationStackInfo contains details related to the stack that runtime validation logic may want to use. type RuntimeValidationStackInfo struct { StackName tokens.QName @@ -245,6 +247,8 @@ type ProgramTestOptions struct { YarnBin string // GoBin is a location of a `go` executable to be run. Taken from the $PATH if missing. GoBin string + // PythonBin is a location of a `python` executable to be run. Taken from the $PATH if missing. + PythonBin string // PipenvBin is a location of a `pipenv` executable to run. Taken from the $PATH if missing. PipenvBin string // DotNetBin is a location of a `dotnet` executable to be run. Taken from the $PATH if missing. @@ -252,6 +256,9 @@ type ProgramTestOptions struct { // Additional environment variables to pass for each command we run. Env []string + + // Automatically create and use a virtual environment, rather than using the Pipenv tool. + UseAutomaticVirtualEnv bool } func (opts *ProgramTestOptions) GetDebugLogLevel() int { @@ -479,14 +486,14 @@ func (rf *regexFlag) Set(v string) error { var directoryMatcher regexFlag var listDirs bool -var pipenvMutex *fsutil.FileMutex +var pipMutex *fsutil.FileMutex func init() { flag.Var(&directoryMatcher, "dirs", "optional list of regexes to use to select integration tests to run") flag.BoolVar(&listDirs, "list-dirs", false, "list available integration tests without running them") - mutexPath := filepath.Join(os.TempDir(), "pipenv-mutex.lock") - pipenvMutex = fsutil.NewFileMutex(mutexPath) + mutexPath := filepath.Join(os.TempDir(), "pip-mutex.lock") + pipMutex = fsutil.NewFileMutex(mutexPath) } // GetLogs retrieves the logs for a given stack in a particular region making the query provided. @@ -628,6 +635,7 @@ type ProgramTester struct { bin string // the `pulumi` binary we are using. yarnBin string // the `yarn` binary we are using. goBin string // the `go` binary we are using. + pythonBin string // the `python` binary we are using. pipenvBin string // The `pipenv` binary we are using. dotNetBin string // the `dotnet` binary we are using. eventLog string // The path to the event log for this test. @@ -668,6 +676,31 @@ func (pt *ProgramTester) getGoBin() (string, error) { return getCmdBin(&pt.goBin, "go", pt.opts.GoBin) } +// getPythonBin returns a path to the currently-installed `python` binary, or an error if it could not be found. +func (pt *ProgramTester) getPythonBin() (string, error) { + if pt.pythonBin == "" { + pt.pythonBin = pt.opts.PythonBin + if pt.opts.PythonBin == "" { + var err error + // Look for "python3" by default, but fallback to `python` if not found as some Python 3 + // distributions (in particular the default python.org Windows installation) do not include + // a `python3` binary. + pythonCmds := []string{"python3", "python"} + for _, bin := range pythonCmds { + pt.pythonBin, err = exec.LookPath(bin) + // Break on the first cmd we find on the path (if any). + if err == nil { + break + } + } + if err != nil { + return "", errors.Wrapf(err, "Expected to find one of %q on $PATH", pythonCmds) + } + } + } + return pt.pythonBin, nil +} + // getPipenvBin returns a path to the currently-installed Pipenv tool, or an error if the tool could not be found. func (pt *ProgramTester) getPipenvBin() (string, error) { return getCmdBin(&pt.pipenvBin, "pipenv", pt.opts.PipenvBin) @@ -703,6 +736,16 @@ func (pt *ProgramTester) yarnCmd(args []string) ([]string, error) { return withOptionalYarnFlags(result), nil } +func (pt *ProgramTester) pythonCmd(args []string) ([]string, error) { + bin, err := pt.getPythonBin() + if err != nil { + return nil, err + } + + cmd := []string{bin} + return append(cmd, args...), nil +} + func (pt *ProgramTester) pipenvCmd(args []string) ([]string, error) { bin, err := pt.getPipenvBin() if err != nil { @@ -737,7 +780,8 @@ func (pt *ProgramTester) runPulumiCommand(name string, args []string, wd string, // the command in the context of the virtual environment that Pipenv created in order to pick up // the correct version of Python. We also need to do this for destroy and refresh so that // dynamic providers are run in the right virtual environment. - if isUpdate { + // This is only necessary when not using automatic virtual environment support. + if !pt.opts.UseAutomaticVirtualEnv && isUpdate { projinfo, err := pt.getProjinfo(wd) if err != nil { return nil @@ -809,6 +853,51 @@ func (pt *ProgramTester) runYarnCommand(name string, args []string, wd string) e return err } +func (pt *ProgramTester) runPythonCommand(name string, args []string, wd string) error { + cmd, err := pt.pythonCmd(args) + if err != nil { + return err + } + + return pt.runCommand(name, cmd, wd) +} + +func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd string) error { + // When installing with `pip install -e`, a PKG-INFO file is created. If two packages are being installed + // this way simultaneously (which happens often, when running tests), both installations will be writing the + // same file simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that + // observed the torn write will fail to install the package. + // + // To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids + // the problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a + // file mutex, so this strategy works even if the go test runner chooses to split up text execution across + // multiple processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd + // need to be sharing the mutex globally in each test process if we weren't using the file system to lock.) + if name == "virtualenv-pip-install-package" { + if err := pipMutex.Lock(); err != nil { + panic(err) + } + + if pt.opts.Verbose { + fprintf(pt.opts.Stdout, "acquired pip install lock\n") + defer fprintf(pt.opts.Stdout, "released pip install lock\n") + } + defer func() { + if err := pipMutex.Unlock(); err != nil { + panic(err) + } + }() + } + + virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0]) + if err != nil { + return err + } + + cmd := append([]string{virtualenvBinPath}, args[1:]...) + return pt.runCommand(name, cmd, wd) +} + func (pt *ProgramTester) runPipenvCommand(name string, args []string, wd string) error { // Pipenv uses setuptools to install and uninstall packages. Setuptools has an installation mode called "develop" // that we use to install the package being tested, since it is 1) lightweight and 2) not doing so has its own set @@ -830,22 +919,22 @@ func (pt *ProgramTester) runPipenvCommand(name string, args []string, wd string) // simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that observed the // torn write will fail to install the package (setuptools crashes). // - // To avoid this problem, we use pipenvMutex to explicitly serialize installation operations. Doing so avoids the - // problem of multiple processes stomping on the same files in the source tree. Note that pipenvMutex is a file + // To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids the + // problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a file // mutex, so this strategy works even if the go test runner chooses to split up text execution across multiple // processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd need to be // sharing the mutex globally in each test process if we weren't using the file system to lock.) if name == "pipenv-install-package" { - if err := pipenvMutex.Lock(); err != nil { + if err := pipMutex.Lock(); err != nil { panic(err) } if pt.opts.Verbose { - fprintf(pt.opts.Stdout, "acquired pipenv install lock\n") - defer fprintf(pt.opts.Stdout, "released pipenv install lock\n") + fprintf(pt.opts.Stdout, "acquired pip install lock\n") + defer fprintf(pt.opts.Stdout, "released pip install lock\n") } defer func() { - if err := pipenvMutex.Unlock(); err != nil { + if err := pipMutex.Unlock(); err != nil { panic(err) } }() @@ -1626,26 +1715,25 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error { return err } - // Create a new Pipenv environment. This bootstraps a new virtual environment containing the version of Python that - // we requested. Note that this version of Python is sourced from the machine, so you must first install the version - // of Python that you are requesting on the host machine before building a virtualenv for it. - pythonVersion := "3" - if runtime.GOOS == "windows" { - // Due to https://bugs.python.org/issue34679, Python Dynamic Providers on Windows do not - // work on Python 3.8.0 (but are fixed in 3.8.1). For now we will force Windows to use 3.7 - // to avoid this bug, until 3.8.1 is available in all our CI systems. - pythonVersion = "3.7" - } - if err = pt.runPipenvCommand("pipenv-new", []string{"--python", pythonVersion}, cwd); err != nil { - return err - } + if pt.opts.UseAutomaticVirtualEnv { + if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil { + return err + } - // Install the package's dependencies. We do this by running `pip` inside the virtualenv that `pipenv` has created. - // We don't use `pipenv install` because we don't want a lock file and prefer the similar model of `pip install` - // which matches what our customers do - err = pt.runPipenvCommand("pipenv-install", []string{"run", "pip", "install", "-r", "requirements.txt"}, cwd) - if err != nil { - return err + projinfo.Proj.Runtime.SetOption("virtualenv", "venv") + projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml") + if err = projinfo.Proj.Save(projfile); err != nil { + return errors.Wrap(err, "saving project") + } + + if err := pt.runVirtualEnvCommand("virtualenv-pip-install", + []string{"pip", "install", "-r", "requirements.txt"}, cwd); err != nil { + return err + } + } else { + if err = pt.preparePythonProjectWithPipenv(cwd); err != nil { + return err + } } if !pt.opts.RunUpdateTest { @@ -1657,6 +1745,31 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error { return nil } +func (pt *ProgramTester) preparePythonProjectWithPipenv(cwd string) error { + // Create a new Pipenv environment. This bootstraps a new virtual environment containing the version of Python that + // we requested. Note that this version of Python is sourced from the machine, so you must first install the version + // of Python that you are requesting on the host machine before building a virtualenv for it. + pythonVersion := "3" + if runtime.GOOS == windowsOS { + // Due to https://bugs.python.org/issue34679, Python Dynamic Providers on Windows do not + // work on Python 3.8.0 (but are fixed in 3.8.1). For now we will force Windows to use 3.7 + // to avoid this bug, until 3.8.1 is available in all our CI systems. + pythonVersion = "3.7" + } + if err := pt.runPipenvCommand("pipenv-new", []string{"--python", pythonVersion}, cwd); err != nil { + return err + } + + // Install the package's dependencies. We do this by running `pip` inside the virtualenv that `pipenv` has created. + // We don't use `pipenv install` because we don't want a lock file and prefer the similar model of `pip install` + // which matches what our customers do + err := pt.runPipenvCommand("pipenv-install", []string{"run", "pip", "install", "-r", "requirements.txt"}, cwd) + if err != nil { + return err + } + return nil +} + // YarnLinkPackageDeps bring in package dependencies via yarn func (pt *ProgramTester) yarnLinkPackageDeps(cwd string) error { for _, dependency := range pt.opts.Dependencies { @@ -1681,15 +1794,33 @@ func (pt *ProgramTester) installPipPackageDeps(cwd string) error { } } - err := pt.runPipenvCommand("pipenv-install-package", []string{"run", "pip", "install", "-e", dep}, cwd) - if err != nil { - return err + if pt.opts.UseAutomaticVirtualEnv { + if err := pt.runVirtualEnvCommand("virtualenv-pip-install-package", + []string{"pip", "install", "-e", dep}, cwd); err != nil { + return err + } + } else { + if err := pt.runPipenvCommand("pipenv-install-package", + []string{"run", "pip", "install", "-e", dep}, cwd); err != nil { + return err + } } } return nil } +func getVirtualenvBinPath(cwd, bin string) (string, error) { + virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin) + if runtime.GOOS == windowsOS { + virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin)) + } + if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() { + return "", errors.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath) + } + return virtualenvBinPath, nil +} + // prepareGoProject runs setup necessary to get a Go project ready for `pulumi` commands. func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error { // Go programs are compiled, so we will compile the project first. @@ -1752,7 +1883,7 @@ func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error { if pt.opts.RunBuild { outBin := filepath.Join(gopath, "bin", string(projinfo.Proj.Name)) - if runtime.GOOS == "windows" { + if runtime.GOOS == windowsOS { outBin = fmt.Sprintf("%s.exe", outBin) } err = pt.runCommand("go-build", []string{goBin, "build", "-o", outBin, "."}, cwd) diff --git a/sdk/go/common/resource/plugin/host.go b/sdk/go/common/resource/plugin/host.go index adea878f7..b04d07b36 100644 --- a/sdk/go/common/resource/plugin/host.go +++ b/sdk/go/common/resource/plugin/host.go @@ -258,7 +258,7 @@ func (host *defaultHost) ListAnalyzers() []Analyzer { func (host *defaultHost) Provider(pkg tokens.Package, version *semver.Version) (Provider, error) { plugin, err := host.loadPlugin(func() (interface{}, error) { // Try to load and bind to a plugin. - plug, err := NewProvider(host, host.ctx, pkg, version) + plug, err := NewProvider(host, host.ctx, pkg, version, host.runtimeOptions) if err == nil && plug != nil { info, infoerr := plug.GetPluginInfo() if infoerr != nil { diff --git a/sdk/go/common/resource/plugin/provider_plugin.go b/sdk/go/common/resource/plugin/provider_plugin.go index 53dcb7c3e..ef4de1caa 100644 --- a/sdk/go/common/resource/plugin/provider_plugin.go +++ b/sdk/go/common/resource/plugin/provider_plugin.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "io" + "os" "strings" "github.com/blang/semver" @@ -64,7 +65,8 @@ type provider struct { // NewProvider attempts to bind to a given package's resource plugin and then creates a gRPC connection to it. If the // plugin could not be found, or an error occurs while creating the child process, an error is returned. -func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Version) (Provider, error) { +func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Version, + options map[string]interface{}) (Provider, error) { // Load the plugin's path by using the standard workspace logic. _, path, err := workspace.GetPluginPath( workspace.ResourcePlugin, strings.Replace(string(pkg), tokens.QNameDelimiter, "_", -1), version) @@ -78,8 +80,14 @@ func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Ve }) } + // Runtime options are passed as environment variables to the provider. + env := os.Environ() + for k, v := range options { + env = append(env, fmt.Sprintf("PULUMI_RUNTIME_%s=%v", strings.ToUpper(k), v)) + } + plug, err := newPlugin(ctx, ctx.Pwd, path, fmt.Sprintf("%v (resource)", pkg), - []string{host.ServerAddr()}, nil /*env*/) + []string{host.ServerAddr()}, env) if err != nil { return nil, err } diff --git a/sdk/python/cmd/pulumi-language-python/main.go b/sdk/python/cmd/pulumi-language-python/main.go index 7fa6ae310..ebfb80a6c 100644 --- a/sdk/python/cmd/pulumi-language-python/main.go +++ b/sdk/python/cmd/pulumi-language-python/main.go @@ -30,6 +30,7 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" "strings" "syscall" @@ -58,7 +59,9 @@ const ( // LanguageRuntimeServer RPC endpoint. func main() { var tracing string + var virtualenv string flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint") + flag.StringVar(&virtualenv, "virtualenv", "", "Virtual environment path to use") // You can use the below flag to request that the language host load a specific executor instead of probing the // PATH. This can be used during testing to override the default location. @@ -102,7 +105,7 @@ func main() { // Fire up a gRPC server, letting the kernel choose a free port. port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{ func(srv *grpc.Server) error { - host := newLanguageHost(pythonExec, engineAddress, tracing) + host := newLanguageHost(pythonExec, engineAddress, tracing, virtualenv) pulumirpc.RegisterLanguageRuntimeServer(srv, host) return nil }, @@ -126,13 +129,15 @@ type pythonLanguageHost struct { exec string engineAddress string tracing string + virtualenv string } -func newLanguageHost(exec, engineAddress, tracing string) pulumirpc.LanguageRuntimeServer { +func newLanguageHost(exec, engineAddress, tracing, virtualenv string) pulumirpc.LanguageRuntimeServer { return &pythonLanguageHost{ exec: exec, engineAddress: engineAddress, tracing: tracing, + virtualenv: virtualenv, } } @@ -161,16 +166,39 @@ func (host *pythonLanguageHost) Run(ctx context.Context, req *pulumirpc.RunReque // Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly. var errResult string - - cmd, err := python.Command(args...) - if err != nil { - return nil, err + var cmd *exec.Cmd + var virtualenv string + if host.virtualenv != "" { + virtualenv = host.virtualenv + if !path.IsAbs(virtualenv) { + cwd, err := os.Getwd() + if err != nil { + return nil, errors.Wrap(err, "getting the working directory") + } + virtualenv = filepath.Join(cwd, virtualenv) + } + if !python.IsVirtualEnv(virtualenv) { + return nil, errors.Errorf("%q doesn't appear to be a virtual environment", virtualenv) + } + cmd = python.VirtualEnvCommand(virtualenv, "python", args...) + } else { + cmd, err = python.Command(args...) + if err != nil { + return nil, err + } } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - if config != "" { - cmd.Env = append(os.Environ(), pulumiConfigVar+"="+config) + if virtualenv != "" || config != "" { + env := os.Environ() + if virtualenv != "" { + env = python.ActivateVirtualEnv(env, virtualenv) + } + if config != "" { + env = append(env, pulumiConfigVar+"="+config) + } + cmd.Env = env } if err := cmd.Run(); err != nil { // Python does not explicitly flush standard out or standard error when exiting abnormally. For this reason, we diff --git a/sdk/python/dist/pulumi-analyzer-policy-python.cmd b/sdk/python/dist/pulumi-analyzer-policy-python.cmd index 1728417b0..8a48ac32c 100755 --- a/sdk/python/dist/pulumi-analyzer-policy-python.cmd +++ b/sdk/python/dist/pulumi-analyzer-policy-python.cmd @@ -5,30 +5,30 @@ set "pulumi_policy_python_engine_address=%1" set "pulumi_policy_python_program=%2" REM Parse the -virtualenv command line argument. -set pulumi_policy_python_virtualenv= +set pulumi_runtime_python_virtualenv= :parse if "%~1"=="" goto endparse if "%~1"=="-virtualenv" ( REM Get the value as a fully-qualified path. - set "pulumi_policy_python_virtualenv=%~f2" + set "pulumi_runtime_python_virtualenv=%~f2" goto endparse ) shift /1 goto parse :endparse -if defined pulumi_policy_python_virtualenv ( +if defined pulumi_runtime_python_virtualenv ( REM If python exists in the virtual environment, set PATH and run it. - if exist "%pulumi_policy_python_virtualenv%\Scripts\python.exe" ( + if exist "%pulumi_runtime_python_virtualenv%\Scripts\python.exe" ( REM Update PATH and unset PYTHONHOME. - set "PATH=%pulumi_policy_python_virtualenv%\Scripts;%PATH%" + set "PATH=%pulumi_runtime_python_virtualenv%\Scripts;%PATH%" set PYTHONHOME= REM Run python from the virtual environment. - "%pulumi_policy_python_virtualenv%\Scripts\python.exe" -u -m pulumi.policy %pulumi_policy_python_engine_address% %pulumi_policy_python_program% + "%pulumi_runtime_python_virtualenv%\Scripts\python.exe" -u -m pulumi.policy %pulumi_policy_python_engine_address% %pulumi_policy_python_program% exit /B ) else ( - echo "%pulumi_policy_python_virtualenv%" doesn't appear to be a virtual environment + echo "%pulumi_runtime_python_virtualenv%" doesn't appear to be a virtual environment exit 1 ) ) else ( diff --git a/sdk/python/dist/pulumi-resource-pulumi-python b/sdk/python/dist/pulumi-resource-pulumi-python index a010a76c0..a2ac54810 100755 --- a/sdk/python/dist/pulumi-resource-pulumi-python +++ b/sdk/python/dist/pulumi-resource-pulumi-python @@ -1,2 +1,31 @@ #!/bin/sh -python3 -u -m pulumi.dynamic $@ \ No newline at end of file + +if [ -n "${PULUMI_RUNTIME_VIRTUALENV:-}" ] ; then + # Remove trailing slash. + PULUMI_RUNTIME_VIRTUALENV=${PULUMI_RUNTIME_VIRTUALENV%/} + + # Make the path absolute (if not already). + case $PULUMI_RUNTIME_VIRTUALENV in + /*) : ;; + *) PULUMI_RUNTIME_VIRTUALENV=$PWD/$PULUMI_RUNTIME_VIRTUALENV;; + esac + + # If python exists in the virtual environment, set PATH and run it. + if [ -f "$PULUMI_RUNTIME_VIRTUALENV/bin/python" ]; then + # Update PATH and unset PYTHONHOME. + PATH="$PULUMI_RUNTIME_VIRTUALENV/bin:$PATH" + export PATH + if [ -n "${PYTHONHOME:-}" ] ; then + unset PYTHONHOME + fi + + # Run python from the virtual environment. + "$PULUMI_RUNTIME_VIRTUALENV/bin/python" -u -m pulumi.dynamic $@ + else + echo "\"$PULUMI_RUNTIME_VIRTUALENV\" doesn't appear to be a virtual environment" + exit 1 + fi +else + # Otherwise, just run python3. + python3 -u -m pulumi.dynamic $@ +fi diff --git a/sdk/python/dist/pulumi-resource-pulumi-python.cmd b/sdk/python/dist/pulumi-resource-pulumi-python.cmd index cf2d1916b..5f220738d 100755 --- a/sdk/python/dist/pulumi-resource-pulumi-python.cmd +++ b/sdk/python/dist/pulumi-resource-pulumi-python.cmd @@ -1,5 +1,21 @@ @echo off -setlocal -REM We use `python` instead of `python3` because Windows Python installers -REM install only `python.exe` by default. -@python -u -m pulumi.dynamic %* \ No newline at end of file + +if defined PULUMI_RUNTIME_VIRTUALENV ( + REM If python exists in the virtual environment, set PATH and run it. + if exist "%PULUMI_RUNTIME_VIRTUALENV%\Scripts\python.exe" ( + REM Update PATH and unset PYTHONHOME. + set "PATH=%PULUMI_RUNTIME_VIRTUALENV%\Scripts;%PATH%" + set PYTHONHOME= + + REM Run python from the virtual environment. + "%PULUMI_RUNTIME_VIRTUALENV%\Scripts\python.exe" -u -m pulumi.dynamic %* + exit /B + ) else ( + echo "%PULUMI_RUNTIME_VIRTUALENV%" doesn't appear to be a virtual environment + exit 1 + ) +) else ( + REM Otherwise, just run python. We use `python` instead of `python3` because Windows + REM Python installers install only `python.exe` by default. + @python -u -m pulumi.dynamic %* +) diff --git a/sdk/python/python.go b/sdk/python/python.go index 2504e3462..1b2afc141 100644 --- a/sdk/python/python.go +++ b/sdk/python/python.go @@ -21,6 +21,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/pkg/errors" ) const windows = "windows" @@ -59,7 +61,7 @@ func Command(arg ...string) (*exec.Cmd, error) { // VirtualEnvCommand returns an *exec.Cmd for running a command from the specified virtual environment // directory. -func VirtualEnvCommand(virtualEnvDir string, name string, arg ...string) *exec.Cmd { +func VirtualEnvCommand(virtualEnvDir, name string, arg ...string) *exec.Cmd { if runtime.GOOS == windows { name = fmt.Sprintf("%s.exe", name) } @@ -67,6 +69,18 @@ func VirtualEnvCommand(virtualEnvDir string, name string, arg ...string) *exec.C return exec.Command(cmdPath, arg...) } +// IsVirtualEnv returns true if the specified directory contains a python binary. +func IsVirtualEnv(dir string) bool { + pyBin := filepath.Join(dir, virtualEnvBinDirName(), "python") + if runtime.GOOS == windows { + pyBin = fmt.Sprintf("%s.exe", pyBin) + } + if info, err := os.Stat(pyBin); err == nil && !info.IsDir() { + return true + } + return false +} + // ActivateVirtualEnv takes an array of environment variables (same format as os.Environ()) and path to // a virtual environment directory, and returns a new "activated" array with the virtual environment's // "bin" dir ("Scripts" on Windows) prepended to the `PATH` environment variable and `PYTHONHOME` variable @@ -96,6 +110,79 @@ func ActivateVirtualEnv(environ []string, virtualEnvDir string) []string { return result } +// InstallDependencies will create a new virtual environment and install dependencies in the root directory. +func InstallDependencies(root string, showOutput bool, saveProj func(virtualenv string) error) error { + if showOutput { + fmt.Println("Creating virtual environment...") + fmt.Println() + } + + // Create the virtual environment by running `python -m venv venv`. + venvDir := filepath.Join(root, "venv") + cmd, err := Command("-m", "venv", venvDir) + if err != nil { + return err + } + if output, err := cmd.CombinedOutput(); err != nil { + if len(output) > 0 { + os.Stdout.Write(output) + fmt.Println() + } + return errors.Wrapf(err, "creating virtual environment at %s", venvDir) + } + + // Save project with venv info. + if err := saveProj("venv"); err != nil { + return err + } + + if showOutput { + fmt.Println("Finished creating virtual environment") + fmt.Println() + } + + // If `requirements.txt` doesn't exist, just exit early. + requirementsPath := filepath.Join(root, "requirements.txt") + if _, err := os.Stat(requirementsPath); os.IsNotExist(err) { + return nil + } + + if showOutput { + fmt.Println("Installing dependencies...") + fmt.Println() + } + + // Install dependencies by running `pip install -r requirements.txt` using the `pip` + // in the virtual environment. + pipCmd := VirtualEnvCommand(venvDir, "pip", "install", "-r", "requirements.txt") + pipCmd.Dir = root + pipCmd.Env = ActivateVirtualEnv(os.Environ(), venvDir) + if showOutput { + // Show stdout/stderr output. + pipCmd.Stdout = os.Stdout + pipCmd.Stderr = os.Stderr + if err := pipCmd.Run(); err != nil { + return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`") + } + } else { + // Otherwise, only show output if there is an error. + if output, err := pipCmd.CombinedOutput(); err != nil { + if len(output) > 0 { + os.Stdout.Write(output) + fmt.Println() + } + return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`") + } + } + + if showOutput { + fmt.Println("Finished installing dependencies") + fmt.Println() + } + + return nil +} + func virtualEnvBinDirName() string { if runtime.GOOS == windows { return "Scripts" diff --git a/sdk/python/python_test.go b/sdk/python/python_test.go index 65de126dd..057fa001d 100644 --- a/sdk/python/python_test.go +++ b/sdk/python/python_test.go @@ -25,6 +25,25 @@ import ( "github.com/stretchr/testify/assert" ) +func TestIsVirtualEnv(t *testing.T) { + // Create a new empty test directory. + tempdir, _ := ioutil.TempDir("", "test-env") + defer os.RemoveAll(tempdir) + + // Assert the empty test directory is not a virtual environment. + assert.False(t, IsVirtualEnv(tempdir)) + + // Create and run a python command to create a virtual environment. + venvDir := filepath.Join(tempdir, "venv") + cmd, err := Command("-m", "venv", venvDir) + assert.NoError(t, err) + err = cmd.Run() + assert.NoError(t, err) + + // Assert the new venv directory is a virtual environment. + assert.True(t, IsVirtualEnv(venvDir)) +} + func TestActivateVirtualEnv(t *testing.T) { venvName := "venv" venvDir := filepath.Join(venvName, "bin") diff --git a/tests/integration/config_basic/python_venv/.gitignore b/tests/integration/config_basic/python_venv/.gitignore new file mode 100644 index 000000000..3f47d8e79 --- /dev/null +++ b/tests/integration/config_basic/python_venv/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/.pulumi/ +/dist/ +/*.egg-info +venv/ diff --git a/tests/integration/config_basic/python_venv/Pulumi.yaml b/tests/integration/config_basic/python_venv/Pulumi.yaml new file mode 100644 index 000000000..7c634ce85 --- /dev/null +++ b/tests/integration/config_basic/python_venv/Pulumi.yaml @@ -0,0 +1,3 @@ +name: config_basic_py +description: A simple Python program that uses configuration. +runtime: python diff --git a/tests/integration/config_basic/python_venv/__main__.py b/tests/integration/config_basic/python_venv/__main__.py new file mode 100644 index 000000000..9f0b25ea2 --- /dev/null +++ b/tests/integration/config_basic/python_venv/__main__.py @@ -0,0 +1,53 @@ +# Copyright 2016-2018, Pulumi Corporation. All rights reserved. + +import pulumi + +# Just test that basic config works. +config = pulumi.Config('config_basic_py') + +# This value is plaintext and doesn't require encryption. +value = config.require('aConfigValue') +assert value == 'this value is a Pythonic value' + +# This value is a secret and is encrypted using the passphrase `supersecret`. +secret = config.require('bEncryptedSecret') +assert secret == 'this super Pythonic secret is encrypted' + +test_data = [ + { + 'key': 'outer', + 'expected_json': '{"inner":"value"}', + 'expected_object': { 'inner': 'value' } + }, + { + 'key': 'names', + 'expected_json': '["a","b","c","super secret name"]', + 'expected_object': ['a', 'b', 'c', 'super secret name'] + }, + { + 'key': 'servers', + 'expected_json': '[{"host":"example","port":80}]', + 'expected_object': [{ 'host': 'example', 'port': 80 }] + }, + { + 'key': 'a', + 'expected_json': '{"b":[{"c":true},{"c":false}]}', + 'expected_object': { 'b': [{ 'c': True }, { 'c': False }] } + }, + { + 'key': 'tokens', + 'expected_json': '["shh"]', + 'expected_object': ['shh'] + }, + { + 'key': 'foo', + 'expected_json': '{"bar":"don\'t tell"}', + 'expected_object': { 'bar': "don't tell" } + } +] + +for test in test_data: + json = config.require(test['key']) + obj = config.require_object(test['key']) + assert json == test['expected_json'] + assert obj == test['expected_object'] diff --git a/tests/integration/config_basic/python_venv/requirements.txt b/tests/integration/config_basic/python_venv/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/dynamic/python_venv/.gitignore b/tests/integration/dynamic/python_venv/.gitignore new file mode 100644 index 000000000..3f47d8e79 --- /dev/null +++ b/tests/integration/dynamic/python_venv/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/.pulumi/ +/dist/ +/*.egg-info +venv/ diff --git a/tests/integration/dynamic/python_venv/Pulumi.yaml b/tests/integration/dynamic/python_venv/Pulumi.yaml new file mode 100644 index 000000000..64e6d7cb6 --- /dev/null +++ b/tests/integration/dynamic/python_venv/Pulumi.yaml @@ -0,0 +1,3 @@ +name: dynamic_py +description: A simple Python program that uses dynamic providers. +runtime: python diff --git a/tests/integration/dynamic/python_venv/__main__.py b/tests/integration/dynamic/python_venv/__main__.py new file mode 100644 index 000000000..9bbd08bc2 --- /dev/null +++ b/tests/integration/dynamic/python_venv/__main__.py @@ -0,0 +1,21 @@ +# Copyright 2016-2018, Pulumi Corporation. All rights reserved. + +import binascii +import os +from pulumi import ComponentResource, export +from pulumi.dynamic import Resource, ResourceProvider, CreateResult + +class RandomResourceProvider(ResourceProvider): + def create(self, props): + val = binascii.b2a_hex(os.urandom(15)).decode("ascii") + return CreateResult(val, { "val": val }) + +class Random(Resource): + val: str + def __init__(self, name, opts = None): + super().__init__(RandomResourceProvider(), name, {"val": ""}, opts) + +r = Random("foo") + +export("random_id", r.id) +export("random_val", r.val) diff --git a/tests/integration/dynamic/python_venv/requirements.txt b/tests/integration/dynamic/python_venv/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/dynamic/python_venv/step1/README.md b/tests/integration/dynamic/python_venv/step1/README.md new file mode 100644 index 000000000..cd4e14144 --- /dev/null +++ b/tests/integration/dynamic/python_venv/step1/README.md @@ -0,0 +1 @@ +Intentionally make no changes. \ No newline at end of file diff --git a/tests/integration/empty/python_venv/.gitignore b/tests/integration/empty/python_venv/.gitignore new file mode 100644 index 000000000..3f47d8e79 --- /dev/null +++ b/tests/integration/empty/python_venv/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/.pulumi/ +/dist/ +/*.egg-info +venv/ diff --git a/tests/integration/empty/python_venv/Pulumi.yaml b/tests/integration/empty/python_venv/Pulumi.yaml new file mode 100644 index 000000000..f4cc153ad --- /dev/null +++ b/tests/integration/empty/python_venv/Pulumi.yaml @@ -0,0 +1,3 @@ +name: emptypy +description: An empty Python Pulumi program. +runtime: python diff --git a/tests/integration/empty/python_venv/__main__.py b/tests/integration/empty/python_venv/__main__.py new file mode 100644 index 000000000..954bbdbeb --- /dev/null +++ b/tests/integration/empty/python_venv/__main__.py @@ -0,0 +1,7 @@ +# Copyright 2016-2018, Pulumi Corporation. All rights reserved. + +def main(): + return None + +if __name__ == "__main__": + main() diff --git a/tests/integration/empty/python_venv/requirements.txt b/tests/integration/empty/python_venv/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 5414d313b..006974dd9 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -76,6 +76,18 @@ func TestEmptyPython(t *testing.T) { }) } +// TestEmptyPythonVenv simply tests that we can run an empty Python project using automatic virtual environment support. +func TestEmptyPythonVenv(t *testing.T) { + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join("empty", "python_venv"), + Dependencies: []string{ + filepath.Join("..", "..", "sdk", "python", "env", "src"), + }, + Quick: true, + UseAutomaticVirtualEnv: true, + }) +} + // TestEmptyGo simply tests that we can build and run an empty Go project. func TestEmptyGo(t *testing.T) { integration.ProgramTest(t, &integration.ProgramTestOptions{ @@ -1087,6 +1099,37 @@ func TestConfigBasicPython(t *testing.T) { }) } +// Tests basic configuration from the perspective of a Pulumi program using automatic virtual environment support. +func TestConfigBasicPythonVenv(t *testing.T) { + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join("config_basic", "python_venv"), + Dependencies: []string{ + filepath.Join("..", "..", "sdk", "python", "env", "src"), + }, + Quick: true, + Config: map[string]string{ + "aConfigValue": "this value is a Pythonic value", + }, + Secrets: map[string]string{ + "bEncryptedSecret": "this super Pythonic secret is encrypted", + }, + OrderedConfig: []integration.ConfigValue{ + {Key: "outer.inner", Value: "value", Path: true}, + {Key: "names[0]", Value: "a", Path: true}, + {Key: "names[1]", Value: "b", Path: true}, + {Key: "names[2]", Value: "c", Path: true}, + {Key: "names[3]", Value: "super secret name", Path: true, Secret: true}, + {Key: "servers[0].port", Value: "80", Path: true}, + {Key: "servers[0].host", Value: "example", Path: true}, + {Key: "a.b[0].c", Value: "true", Path: true}, + {Key: "a.b[1].c", Value: "false", Path: true}, + {Key: "tokens[0]", Value: "shh", Path: true, Secret: true}, + {Key: "foo.bar", Value: "don't tell", Path: true, Secret: true}, + }, + UseAutomaticVirtualEnv: true, + }) +} + // Tests basic configuration from the perspective of a Pulumi Go program. func TestConfigBasicGo(t *testing.T) { integration.ProgramTest(t, &integration.ProgramTestOptions{ @@ -1446,6 +1489,28 @@ func TestDynamicPython(t *testing.T) { }) } +// Tests dynamic provider in Python using automatic virtual environment support. +func TestDynamicPythonVenv(t *testing.T) { + var randomVal string + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join("dynamic", "python_venv"), + Dependencies: []string{ + filepath.Join("..", "..", "sdk", "python", "env", "src"), + }, + ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { + randomVal = stack.Outputs["random_val"].(string) + }, + EditDirs: []integration.EditDir{{ + Dir: "step1", + Additive: true, + ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { + assert.Equal(t, randomVal, stack.Outputs["random_val"].(string)) + }, + }}, + UseAutomaticVirtualEnv: true, + }) +} + func TestResourceWithSecretSerialization(t *testing.T) { integration.ProgramTest(t, &integration.ProgramTestOptions{ Dir: "secret_outputs",