maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Install and use dependencies automatically for new Python projects (#4775)

Browse source 
Automatically create a virtual environment and install dependencies in it with `pulumi new` and `pulumi policy new` for Python templates.

This will save a new `virtualenv` runtime option in `Pulumi.yaml` (`PulumiPolicy.yaml` for policy packs):

```yaml
runtime:
  name: python
  options:
    virtualenv: venv
```

`virtualenv` is the path to a virtual environment that Pulumi will use when running `python` commands.

Existing projects are unaffected and can opt-in to using this by setting `virtualenv`, otherwise, they'll continue to work as-is.
This commit is contained in:
Justin Van Patten 2020-06-09 23:42:53 +00:00 committed by GitHub
parent 238adf2f2f
commit b77ec919d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 608 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -14,6 +14,9 @@ CHANGELOG
- Allow users to specify base64 encoded strings as GOOGLE_CREDENTIALS
[#4773](https://github.com/pulumi/pulumi/pull/4773)
- Install and use dependencies automatically for new Python projects.
[#4775](https://github.com/pulumi/pulumi/pull/4775)
---
## 2.3.0 (2020-05-27)

41
pkg/backend/httpstate/policypack.go
View file

@ -320,42 +320,15 @@ func completeNodeJSInstall(finalDir string) error {
}
func completePythonInstall(finalDir, projPath string, proj *workspace.PolicyPackProject) error {
// Create virtual environment.
venvDir := filepath.Join(finalDir, "venv")
cmd, err := python.Command("-m", "venv", venvDir)
if err != nil {
return err
}
if output, err := cmd.CombinedOutput(); err != nil {
if len(output) > 0 {
os.Stdout.Write(output)
fmt.Println()
if err := python.InstallDependencies(finalDir, false /*showOutput*/, func(virtualenv string) error {
// Save project with venv info.
proj.Runtime.SetOption("virtualenv", virtualenv)
if err := proj.Save(projPath); err != nil {
return errors.Wrapf(err, "saving project at %s", projPath)
}
return errors.Wrapf(err, "creating virtual environment at %s", venvDir)
}
// Save project with venv info.
proj.Runtime.SetOption("virtualenv", "venv")
if err := proj.Save(projPath); err != nil {
return errors.Wrapf(err, "saving project at %s", projPath)
}
requirementsPath := filepath.Join(finalDir, "requirements.txt")
if _, err := os.Stat(requirementsPath); os.IsNotExist(err) {
return nil
}
pipCmd := python.VirtualEnvCommand(venvDir, "pip", "install", "-r", "requirements.txt")
pipCmd.Dir = finalDir
pipCmd.Env = python.ActivateVirtualEnv(os.Environ(), venvDir)
if output, err := pipCmd.CombinedOutput(); err != nil {
if len(output) > 0 {
os.Stdout.Write(output)
fmt.Println()
}
return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`")
}); err != nil {
return err
}
fmt.Println("Finished installing policy pack")

33
pkg/cmd/pulumi/new.go
View file

@ -46,6 +46,7 @@ import (
"github.com/pulumi/pulumi/sdk/v2/go/common/util/executable"
"github.com/pulumi/pulumi/sdk/v2/go/common/util/logging"
"github.com/pulumi/pulumi/sdk/v2/go/common/workspace"
"github.com/pulumi/pulumi/sdk/v2/python"
)
type promptForValueFunc func(yes bool, valueType string, defaultValue string, secret bool,
@ -568,6 +569,8 @@ func installDependencies(proj *workspace.Project, root string) error {
return errors.Wrapf(err, "%s install failed; rerun manually to try again, "+
"then run 'pulumi up' to perform an initial deployment", bin)
}
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
return pythonInstallDependencies(proj, root)
} else if strings.EqualFold(proj.Runtime.Name(), "dotnet") {
return dotnetInstallDependenciesAndBuild(proj, root)
} else if strings.EqualFold(proj.Runtime.Name(), "go") {
@ -597,6 +600,18 @@ func nodeInstallDependencies() (string, error) {
return bin, nil
}
// pythonInstallDependencies will create a new virtual environment and install dependencies.
func pythonInstallDependencies(proj *workspace.Project, root string) error {
return python.InstallDependencies(root, true /*showOutput*/, func(virtualenv string) error {
// Save project with venv info.
proj.Runtime.SetOption("virtualenv", virtualenv)
if err := workspace.SaveProject(proj); err != nil {
return errors.Wrap(err, "saving project")
}
return nil
})
}
// dotnetInstallDependenciesAndBuild will install dependencies and build the project.
func dotnetInstallDependenciesAndBuild(proj *workspace.Project, root string) error {
contract.Assert(proj != nil)
@ -672,18 +687,14 @@ func printNextSteps(proj *workspace.Project, originalCwd, cwd string, generateOn
commands = append(commands, cd)
}
if strings.EqualFold(proj.Runtime.Name(), "nodejs") && generateOnly {
// If we're generating a NodeJS project, and we didn't install dependencies (generateOnly),
// instruct the user to do so.
commands = append(commands, "npm install")
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
// If we're generating a Python project, instruct the user to set up and activate a virtual
// environment.
commands = append(commands, pythonCommands()...)
}
// If we didn't create a stack, show that as a command to run before `pulumi up`.
if generateOnly {
// We didn't install dependencies, so instruct the user to do so.
if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
commands = append(commands, "npm install")
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
commands = append(commands, pythonCommands()...)
}
// We didn't create a stack so show that as a command to run before `pulumi up`.
commands = append(commands, "pulumi stack init")
}

42
pkg/cmd/pulumi/policy_new.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pulumi/pulumi/sdk/v2/go/common/util/cmdutil"
"github.com/pulumi/pulumi/sdk/v2/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v2/go/common/workspace"
"github.com/pulumi/pulumi/sdk/v2/python"
"github.com/spf13/cobra"
survey "gopkg.in/AlecAivazis/survey.v1"
surveycore "gopkg.in/AlecAivazis/survey.v1/core"
@ -162,14 +163,14 @@ func runNewPolicyPack(args newPolicyArgs) error {
fmt.Println("Created Policy Pack!")
proj, root, err := readPolicyProject()
proj, projPath, root, err := readPolicyProject()
if err != nil {
return err
}
// Install dependencies.
if !args.generateOnly {
if err := installPolicyPackDependencies(proj); err != nil {
if err := installPolicyPackDependencies(proj, projPath, root); err != nil {
return err
}
}
@ -185,26 +186,36 @@ func runNewPolicyPack(args newPolicyArgs) error {
return nil
}
func installPolicyPackDependencies(proj *workspace.PolicyPackProject) error {
func installPolicyPackDependencies(proj *workspace.PolicyPackProject, projPath, root string) error {
// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
if bin, err := nodeInstallDependencies(); err != nil {
return errors.Wrapf(err, "`%s install` failed; rerun manually to try again.", bin)
}
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
if err := python.InstallDependencies(root, true /*showOutput*/, func(virtualenv string) error {
// Save project with venv info.
proj.Runtime.SetOption("virtualenv", virtualenv)
if err := proj.Save(projPath); err != nil {
return errors.Wrapf(err, "saving project at %s", projPath)
}
return nil
}); err != nil {
return err
}
}
return nil
}
func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, generateOnly bool, opts display.Options) {
var commands []string
if strings.EqualFold(proj.Runtime.Name(), "nodejs") && generateOnly {
// If we're generating a NodeJS policy pack, and we didn't install dependencies
// (generateOnly), instruct the user to do so.
commands = append(commands, "npm install")
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
// If we're generating a Python policy pack, instruct the user to set up and
// activate a virtual environment.
commands = append(commands, pythonCommands()...)
if generateOnly {
// We didn't install dependencies, so instruct the user to do so.
if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
commands = append(commands, "npm install")
} else if strings.EqualFold(proj.Runtime.Name(), "python") {
commands = append(commands, pythonCommands()...)
}
}
if len(commands) == 1 {
@ -228,8 +239,7 @@ func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, ge
[]string{"run the Policy Pack against a Pulumi program, in the directory of the Pulumi program run"}
usageCommands := []string{fmt.Sprintf("pulumi up --policy-pack %s", root)}
// Currently, only Node.js Policy Packs can be published.
if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
if strings.EqualFold(proj.Runtime.Name(), "nodejs") || strings.EqualFold(proj.Runtime.Name(), "python") {
usageCommandPreambles = append(usageCommandPreambles, "publish the Policy Pack, run")
usageCommands = append(usageCommands, "pulumi policy publish [org-name]")
}
@ -251,12 +261,6 @@ func printPolicyPackNextSteps(proj *workspace.PolicyPackProject, root string, ge
}
fmt.Println()
}
// Add special note for Python.
if strings.EqualFold(proj.Runtime.Name(), "python") {
fmt.Println("Note: When running the Policy Pack against a Pulumi program, if the Pulumi program is " +
"also Python, both the Pulumi program and Policy Pack must use the same virtual environment.")
}
}
// choosePolicyPackTemplate will prompt the user to choose amongst the available templates.

2
pkg/cmd/pulumi/policy_publish.go
View file

@ -69,7 +69,7 @@ func newPolicyPublishCmd() *cobra.Command {
// Load metadata about the current project.
//
proj, root, err := readPolicyProject()
proj, _, root, err := readPolicyProject()
if err != nil {
return err
}

12
pkg/cmd/pulumi/util.go
View file

@ -402,26 +402,26 @@ func readProject() (*workspace.Project, string, error) {
// readPolicyProject attempts to detect and read a Pulumi PolicyPack project for the current
// workspace. If the project is successfully detected and read, it is returned along with the path
// to its containing directory, which will be used as the root of the project's Pulumi program.
func readPolicyProject() (*workspace.PolicyPackProject, string, error) {
func readPolicyProject() (*workspace.PolicyPackProject, string, string, error) {
pwd, err := os.Getwd()
if err != nil {
return nil, "", err
return nil, "", "", err
}
// Now that we got here, we have a path, so we will try to load it.
path, err := workspace.DetectPolicyPackPathFrom(pwd)
if err != nil {
return nil, "", errors.Wrapf(err, "failed to find current Pulumi project because of "+
return nil, "", "", errors.Wrapf(err, "failed to find current Pulumi project because of "+
"an error when searching for the PulumiPolicy.yaml file (searching upwards from %s)", pwd)
} else if path == "" {
return nil, "", fmt.Errorf("no PulumiPolicy.yaml project file found (searching upwards from %s)", pwd)
return nil, "", "", fmt.Errorf("no PulumiPolicy.yaml project file found (searching upwards from %s)", pwd)
}
proj, err := workspace.LoadPolicyPack(path)
if err != nil {
return nil, "", errors.Wrapf(err, "failed to load Pulumi policy project located at %q", path)
return nil, "", "", errors.Wrapf(err, "failed to load Pulumi policy project located at %q", path)
}
return proj, filepath.Dir(path), nil
return proj, path, filepath.Dir(path), nil
}
// anyWriter is an io.Writer that will set itself to `true` iff any call to `anyWriter.Write` is made with a

197
pkg/testing/integration/program.go
View file

@ -60,6 +60,8 @@ const NodeJSRuntime = "nodejs"
const GoRuntime = "go"
const DotNetRuntime = "dotnet"
const windowsOS = "windows"
// RuntimeValidationStackInfo contains details related to the stack that runtime validation logic may want to use.
type RuntimeValidationStackInfo struct {
StackName tokens.QName
@ -245,6 +247,8 @@ type ProgramTestOptions struct {
YarnBin string
// GoBin is a location of a `go` executable to be run. Taken from the $PATH if missing.
GoBin string
// PythonBin is a location of a `python` executable to be run. Taken from the $PATH if missing.
PythonBin string
// PipenvBin is a location of a `pipenv` executable to run. Taken from the $PATH if missing.
PipenvBin string
// DotNetBin is a location of a `dotnet` executable to be run. Taken from the $PATH if missing.
@ -252,6 +256,9 @@ type ProgramTestOptions struct {
// Additional environment variables to pass for each command we run.
Env []string
// Automatically create and use a virtual environment, rather than using the Pipenv tool.
UseAutomaticVirtualEnv bool
}
func (opts *ProgramTestOptions) GetDebugLogLevel() int {
@ -479,14 +486,14 @@ func (rf *regexFlag) Set(v string) error {
var directoryMatcher regexFlag
var listDirs bool
var pipenvMutex *fsutil.FileMutex
var pipMutex *fsutil.FileMutex
func init() {
flag.Var(&directoryMatcher, "dirs", "optional list of regexes to use to select integration tests to run")
flag.BoolVar(&listDirs, "list-dirs", false, "list available integration tests without running them")
mutexPath := filepath.Join(os.TempDir(), "pipenv-mutex.lock")
pipenvMutex = fsutil.NewFileMutex(mutexPath)
mutexPath := filepath.Join(os.TempDir(), "pip-mutex.lock")
pipMutex = fsutil.NewFileMutex(mutexPath)
}
// GetLogs retrieves the logs for a given stack in a particular region making the query provided.
@ -628,6 +635,7 @@ type ProgramTester struct {
bin string // the `pulumi` binary we are using.
yarnBin string // the `yarn` binary we are using.
goBin string // the `go` binary we are using.
pythonBin string // the `python` binary we are using.
pipenvBin string // The `pipenv` binary we are using.
dotNetBin string // the `dotnet` binary we are using.
eventLog string // The path to the event log for this test.
@ -668,6 +676,31 @@ func (pt *ProgramTester) getGoBin() (string, error) {
return getCmdBin(&pt.goBin, "go", pt.opts.GoBin)
}
// getPythonBin returns a path to the currently-installed `python` binary, or an error if it could not be found.
func (pt *ProgramTester) getPythonBin() (string, error) {
if pt.pythonBin == "" {
pt.pythonBin = pt.opts.PythonBin
if pt.opts.PythonBin == "" {
var err error
// Look for "python3" by default, but fallback to `python` if not found as some Python 3
// distributions (in particular the default python.org Windows installation) do not include
// a `python3` binary.
pythonCmds := []string{"python3", "python"}
for _, bin := range pythonCmds {
pt.pythonBin, err = exec.LookPath(bin)
// Break on the first cmd we find on the path (if any).
if err == nil {
break
}
}
if err != nil {
return "", errors.Wrapf(err, "Expected to find one of %q on $PATH", pythonCmds)
}
}
}
return pt.pythonBin, nil
}
// getPipenvBin returns a path to the currently-installed Pipenv tool, or an error if the tool could not be found.
func (pt *ProgramTester) getPipenvBin() (string, error) {
return getCmdBin(&pt.pipenvBin, "pipenv", pt.opts.PipenvBin)
@ -703,6 +736,16 @@ func (pt *ProgramTester) yarnCmd(args []string) ([]string, error) {
return withOptionalYarnFlags(result), nil
}
func (pt *ProgramTester) pythonCmd(args []string) ([]string, error) {
bin, err := pt.getPythonBin()
if err != nil {
return nil, err
}
cmd := []string{bin}
return append(cmd, args...), nil
}
func (pt *ProgramTester) pipenvCmd(args []string) ([]string, error) {
bin, err := pt.getPipenvBin()
if err != nil {
@ -737,7 +780,8 @@ func (pt *ProgramTester) runPulumiCommand(name string, args []string, wd string,
// the command in the context of the virtual environment that Pipenv created in order to pick up
// the correct version of Python. We also need to do this for destroy and refresh so that
// dynamic providers are run in the right virtual environment.
if isUpdate {
// This is only necessary when not using automatic virtual environment support.
if !pt.opts.UseAutomaticVirtualEnv && isUpdate {
projinfo, err := pt.getProjinfo(wd)
if err != nil {
return nil
@ -809,6 +853,51 @@ func (pt *ProgramTester) runYarnCommand(name string, args []string, wd string) e
return err
}
func (pt *ProgramTester) runPythonCommand(name string, args []string, wd string) error {
cmd, err := pt.pythonCmd(args)
if err != nil {
return err
}
return pt.runCommand(name, cmd, wd)
}
func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd string) error {
// When installing with `pip install -e`, a PKG-INFO file is created. If two packages are being installed
// this way simultaneously (which happens often, when running tests), both installations will be writing the
// same file simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that
// observed the torn write will fail to install the package.
//
// To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids
// the problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a
// file mutex, so this strategy works even if the go test runner chooses to split up text execution across
// multiple processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd
// need to be sharing the mutex globally in each test process if we weren't using the file system to lock.)
if name == "virtualenv-pip-install-package" {
if err := pipMutex.Lock(); err != nil {
panic(err)
}
if pt.opts.Verbose {
fprintf(pt.opts.Stdout, "acquired pip install lock\n")
defer fprintf(pt.opts.Stdout, "released pip install lock\n")
}
defer func() {
if err := pipMutex.Unlock(); err != nil {
panic(err)
}
}()
}
virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0])
if err != nil {
return err
}
cmd := append([]string{virtualenvBinPath}, args[1:]...)
return pt.runCommand(name, cmd, wd)
}
func (pt *ProgramTester) runPipenvCommand(name string, args []string, wd string) error {
// Pipenv uses setuptools to install and uninstall packages. Setuptools has an installation mode called "develop"
// that we use to install the package being tested, since it is 1) lightweight and 2) not doing so has its own set
@ -830,22 +919,22 @@ func (pt *ProgramTester) runPipenvCommand(name string, args []string, wd string)
// simultaneously. If one process catches "PKG-INFO" in a half-written state, the one process that observed the
// torn write will fail to install the package (setuptools crashes).
//
// To avoid this problem, we use pipenvMutex to explicitly serialize installation operations. Doing so avoids the
// problem of multiple processes stomping on the same files in the source tree. Note that pipenvMutex is a file
// To avoid this problem, we use pipMutex to explicitly serialize installation operations. Doing so avoids the
// problem of multiple processes stomping on the same files in the source tree. Note that pipMutex is a file
// mutex, so this strategy works even if the go test runner chooses to split up text execution across multiple
// processes. (Furthermore, each test gets an instance of ProgramTester and thus the mutex, so we'd need to be
// sharing the mutex globally in each test process if we weren't using the file system to lock.)
if name == "pipenv-install-package" {
if err := pipenvMutex.Lock(); err != nil {
if err := pipMutex.Lock(); err != nil {
panic(err)
}
if pt.opts.Verbose {
fprintf(pt.opts.Stdout, "acquired pipenv install lock\n")
defer fprintf(pt.opts.Stdout, "released pipenv install lock\n")
fprintf(pt.opts.Stdout, "acquired pip install lock\n")
defer fprintf(pt.opts.Stdout, "released pip install lock\n")
}
defer func() {
if err := pipenvMutex.Unlock(); err != nil {
if err := pipMutex.Unlock(); err != nil {
panic(err)
}
}()
@ -1626,26 +1715,25 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error {
return err
}
// Create a new Pipenv environment. This bootstraps a new virtual environment containing the version of Python that
// we requested. Note that this version of Python is sourced from the machine, so you must first install the version
// of Python that you are requesting on the host machine before building a virtualenv for it.
pythonVersion := "3"
if runtime.GOOS == "windows" {
// Due to https://bugs.python.org/issue34679, Python Dynamic Providers on Windows do not
// work on Python 3.8.0 (but are fixed in 3.8.1). For now we will force Windows to use 3.7
// to avoid this bug, until 3.8.1 is available in all our CI systems.
pythonVersion = "3.7"
}
if err = pt.runPipenvCommand("pipenv-new", []string{"--python", pythonVersion}, cwd); err != nil {
return err
}
if pt.opts.UseAutomaticVirtualEnv {
if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil {
return err
}
// Install the package's dependencies. We do this by running `pip` inside the virtualenv that `pipenv` has created.
// We don't use `pipenv install` because we don't want a lock file and prefer the similar model of `pip install`
// which matches what our customers do
err = pt.runPipenvCommand("pipenv-install", []string{"run", "pip", "install", "-r", "requirements.txt"}, cwd)
if err != nil {
return err
projinfo.Proj.Runtime.SetOption("virtualenv", "venv")
projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml")
if err = projinfo.Proj.Save(projfile); err != nil {
return errors.Wrap(err, "saving project")
}
if err := pt.runVirtualEnvCommand("virtualenv-pip-install",
[]string{"pip", "install", "-r", "requirements.txt"}, cwd); err != nil {
return err
}
} else {
if err = pt.preparePythonProjectWithPipenv(cwd); err != nil {
return err
}
}
if !pt.opts.RunUpdateTest {
@ -1657,6 +1745,31 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error {
return nil
}
func (pt *ProgramTester) preparePythonProjectWithPipenv(cwd string) error {
// Create a new Pipenv environment. This bootstraps a new virtual environment containing the version of Python that
// we requested. Note that this version of Python is sourced from the machine, so you must first install the version
// of Python that you are requesting on the host machine before building a virtualenv for it.
pythonVersion := "3"
if runtime.GOOS == windowsOS {
// Due to https://bugs.python.org/issue34679, Python Dynamic Providers on Windows do not
// work on Python 3.8.0 (but are fixed in 3.8.1). For now we will force Windows to use 3.7
// to avoid this bug, until 3.8.1 is available in all our CI systems.
pythonVersion = "3.7"
}
if err := pt.runPipenvCommand("pipenv-new", []string{"--python", pythonVersion}, cwd); err != nil {
return err
}
// Install the package's dependencies. We do this by running `pip` inside the virtualenv that `pipenv` has created.
// We don't use `pipenv install` because we don't want a lock file and prefer the similar model of `pip install`
// which matches what our customers do
err := pt.runPipenvCommand("pipenv-install", []string{"run", "pip", "install", "-r", "requirements.txt"}, cwd)
if err != nil {
return err
}
return nil
}
// YarnLinkPackageDeps bring in package dependencies via yarn
func (pt *ProgramTester) yarnLinkPackageDeps(cwd string) error {
for _, dependency := range pt.opts.Dependencies {
@ -1681,15 +1794,33 @@ func (pt *ProgramTester) installPipPackageDeps(cwd string) error {
}
}
err := pt.runPipenvCommand("pipenv-install-package", []string{"run", "pip", "install", "-e", dep}, cwd)
if err != nil {
return err
if pt.opts.UseAutomaticVirtualEnv {
if err := pt.runVirtualEnvCommand("virtualenv-pip-install-package",
[]string{"pip", "install", "-e", dep}, cwd); err != nil {
return err
}
} else {
if err := pt.runPipenvCommand("pipenv-install-package",
[]string{"run", "pip", "install", "-e", dep}, cwd); err != nil {
return err
}
}
}
return nil
}
func getVirtualenvBinPath(cwd, bin string) (string, error) {
virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin)
if runtime.GOOS == windowsOS {
virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin))
}
if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() {
return "", errors.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath)
}
return virtualenvBinPath, nil
}
// prepareGoProject runs setup necessary to get a Go project ready for `pulumi` commands.
func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error {
// Go programs are compiled, so we will compile the project first.
@ -1752,7 +1883,7 @@ func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error {
if pt.opts.RunBuild {
outBin := filepath.Join(gopath, "bin", string(projinfo.Proj.Name))
if runtime.GOOS == "windows" {
if runtime.GOOS == windowsOS {
outBin = fmt.Sprintf("%s.exe", outBin)
}
err = pt.runCommand("go-build", []string{goBin, "build", "-o", outBin, "."}, cwd)

2
sdk/go/common/resource/plugin/host.go
View file

@ -258,7 +258,7 @@ func (host *defaultHost) ListAnalyzers() []Analyzer {
func (host *defaultHost) Provider(pkg tokens.Package, version *semver.Version) (Provider, error) {
plugin, err := host.loadPlugin(func() (interface{}, error) {
// Try to load and bind to a plugin.
plug, err := NewProvider(host, host.ctx, pkg, version)
plug, err := NewProvider(host, host.ctx, pkg, version, host.runtimeOptions)
if err == nil && plug != nil {
info, infoerr := plug.GetPluginInfo()
if infoerr != nil {

12
sdk/go/common/resource/plugin/provider_plugin.go
View file

@ -18,6 +18,7 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"github.com/blang/semver"
@ -64,7 +65,8 @@ type provider struct {
// NewProvider attempts to bind to a given package's resource plugin and then creates a gRPC connection to it. If the
// plugin could not be found, or an error occurs while creating the child process, an error is returned.
func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Version) (Provider, error) {
func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Version,
options map[string]interface{}) (Provider, error) {
// Load the plugin's path by using the standard workspace logic.
_, path, err := workspace.GetPluginPath(
workspace.ResourcePlugin, strings.Replace(string(pkg), tokens.QNameDelimiter, "_", -1), version)
@ -78,8 +80,14 @@ func NewProvider(host Host, ctx *Context, pkg tokens.Package, version *semver.Ve
})
}
// Runtime options are passed as environment variables to the provider.
env := os.Environ()
for k, v := range options {
env = append(env, fmt.Sprintf("PULUMI_RUNTIME_%s=%v", strings.ToUpper(k), v))
}
plug, err := newPlugin(ctx, ctx.Pwd, path, fmt.Sprintf("%v (resource)", pkg),
[]string{host.ServerAddr()}, nil /*env*/)
[]string{host.ServerAddr()}, env)
if err != nil {
return nil, err
}

44
sdk/python/cmd/pulumi-language-python/main.go
View file

@ -30,6 +30,7 @@ import (
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"syscall"
@ -58,7 +59,9 @@ const (
// LanguageRuntimeServer RPC endpoint.
func main() {
var tracing string
var virtualenv string
flag.StringVar(&tracing, "tracing", "", "Emit tracing to a Zipkin-compatible tracing endpoint")
flag.StringVar(&virtualenv, "virtualenv", "", "Virtual environment path to use")
// You can use the below flag to request that the language host load a specific executor instead of probing the
// PATH. This can be used during testing to override the default location.
@ -102,7 +105,7 @@ func main() {
// Fire up a gRPC server, letting the kernel choose a free port.
port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{
func(srv *grpc.Server) error {
host := newLanguageHost(pythonExec, engineAddress, tracing)
host := newLanguageHost(pythonExec, engineAddress, tracing, virtualenv)
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
return nil
},
@ -126,13 +129,15 @@ type pythonLanguageHost struct {
exec string
engineAddress string
tracing string
virtualenv string
}
func newLanguageHost(exec, engineAddress, tracing string) pulumirpc.LanguageRuntimeServer {
func newLanguageHost(exec, engineAddress, tracing, virtualenv string) pulumirpc.LanguageRuntimeServer {
return &pythonLanguageHost{
exec: exec,
engineAddress: engineAddress,
tracing: tracing,
virtualenv: virtualenv,
}
}
@ -161,16 +166,39 @@ func (host *pythonLanguageHost) Run(ctx context.Context, req *pulumirpc.RunReque
// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
var errResult string
cmd, err := python.Command(args...)
if err != nil {
return nil, err
var cmd *exec.Cmd
var virtualenv string
if host.virtualenv != "" {
virtualenv = host.virtualenv
if !path.IsAbs(virtualenv) {
cwd, err := os.Getwd()
if err != nil {
return nil, errors.Wrap(err, "getting the working directory")
}
virtualenv = filepath.Join(cwd, virtualenv)
}
if !python.IsVirtualEnv(virtualenv) {
return nil, errors.Errorf("%q doesn't appear to be a virtual environment", virtualenv)
}
cmd = python.VirtualEnvCommand(virtualenv, "python", args...)
} else {
cmd, err = python.Command(args...)
if err != nil {
return nil, err
}
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if config != "" {
cmd.Env = append(os.Environ(), pulumiConfigVar+"="+config)
if virtualenv != "" || config != "" {
env := os.Environ()
if virtualenv != "" {
env = python.ActivateVirtualEnv(env, virtualenv)
}
if config != "" {
env = append(env, pulumiConfigVar+"="+config)
}
cmd.Env = env
}
if err := cmd.Run(); err != nil {
// Python does not explicitly flush standard out or standard error when exiting abnormally. For this reason, we

14
sdk/python/dist/pulumi-analyzer-policy-python.cmd vendored
View file

@ -5,30 +5,30 @@ set "pulumi_policy_python_engine_address=%1"
set "pulumi_policy_python_program=%2"
REM Parse the -virtualenv command line argument.
set pulumi_policy_python_virtualenv=
set pulumi_runtime_python_virtualenv=
:parse
if "%~1"=="" goto endparse
if "%~1"=="-virtualenv" (
REM Get the value as a fully-qualified path.
set "pulumi_policy_python_virtualenv=%~f2"
set "pulumi_runtime_python_virtualenv=%~f2"
goto endparse
)
shift /1
goto parse
:endparse
if defined pulumi_policy_python_virtualenv (
if defined pulumi_runtime_python_virtualenv (
REM If python exists in the virtual environment, set PATH and run it.
if exist "%pulumi_policy_python_virtualenv%\Scripts\python.exe" (
if exist "%pulumi_runtime_python_virtualenv%\Scripts\python.exe" (
REM Update PATH and unset PYTHONHOME.
set "PATH=%pulumi_policy_python_virtualenv%\Scripts;%PATH%"
set "PATH=%pulumi_runtime_python_virtualenv%\Scripts;%PATH%"
set PYTHONHOME=
REM Run python from the virtual environment.
"%pulumi_policy_python_virtualenv%\Scripts\python.exe" -u -m pulumi.policy %pulumi_policy_python_engine_address% %pulumi_policy_python_program%
"%pulumi_runtime_python_virtualenv%\Scripts\python.exe" -u -m pulumi.policy %pulumi_policy_python_engine_address% %pulumi_policy_python_program%
exit /B
) else (
echo "%pulumi_policy_python_virtualenv%" doesn't appear to be a virtual environment
echo "%pulumi_runtime_python_virtualenv%" doesn't appear to be a virtual environment
exit 1
)
) else (

31
sdk/python/dist/pulumi-resource-pulumi-python vendored
View file

@ -1,2 +1,31 @@
#!/bin/sh
python3 -u -m pulumi.dynamic $@
if [ -n "${PULUMI_RUNTIME_VIRTUALENV:-}" ] ; then
# Remove trailing slash.
PULUMI_RUNTIME_VIRTUALENV=${PULUMI_RUNTIME_VIRTUALENV%/}
# Make the path absolute (if not already).
case $PULUMI_RUNTIME_VIRTUALENV in
/*) : ;;
*) PULUMI_RUNTIME_VIRTUALENV=$PWD/$PULUMI_RUNTIME_VIRTUALENV;;
esac
# If python exists in the virtual environment, set PATH and run it.
if [ -f "$PULUMI_RUNTIME_VIRTUALENV/bin/python" ]; then
# Update PATH and unset PYTHONHOME.
PATH="$PULUMI_RUNTIME_VIRTUALENV/bin:$PATH"
export PATH
if [ -n "${PYTHONHOME:-}" ] ; then
unset PYTHONHOME
fi
# Run python from the virtual environment.
"$PULUMI_RUNTIME_VIRTUALENV/bin/python" -u -m pulumi.dynamic $@
else
echo "\"$PULUMI_RUNTIME_VIRTUALENV\" doesn't appear to be a virtual environment"
exit 1
fi
else
# Otherwise, just run python3.
python3 -u -m pulumi.dynamic $@
fi

24
sdk/python/dist/pulumi-resource-pulumi-python.cmd vendored
View file

@ -1,5 +1,21 @@
@echo off
setlocal
REM We use `python` instead of `python3` because Windows Python installers
REM install only `python.exe` by default.
@python -u -m pulumi.dynamic %*
if defined PULUMI_RUNTIME_VIRTUALENV (
REM If python exists in the virtual environment, set PATH and run it.
if exist "%PULUMI_RUNTIME_VIRTUALENV%\Scripts\python.exe" (
REM Update PATH and unset PYTHONHOME.
set "PATH=%PULUMI_RUNTIME_VIRTUALENV%\Scripts;%PATH%"
set PYTHONHOME=
REM Run python from the virtual environment.
"%PULUMI_RUNTIME_VIRTUALENV%\Scripts\python.exe" -u -m pulumi.dynamic %*
exit /B
) else (
echo "%PULUMI_RUNTIME_VIRTUALENV%" doesn't appear to be a virtual environment
exit 1
)
) else (
REM Otherwise, just run python. We use `python` instead of `python3` because Windows
REM Python installers install only `python.exe` by default.
@python -u -m pulumi.dynamic %*
)

89
sdk/python/python.go
View file

@ -21,6 +21,8 @@ import (
"path/filepath"
"runtime"
"strings"
"github.com/pkg/errors"
)
const windows = "windows"
@ -59,7 +61,7 @@ func Command(arg ...string) (*exec.Cmd, error) {
// VirtualEnvCommand returns an *exec.Cmd for running a command from the specified virtual environment
// directory.
func VirtualEnvCommand(virtualEnvDir string, name string, arg ...string) *exec.Cmd {
func VirtualEnvCommand(virtualEnvDir, name string, arg ...string) *exec.Cmd {
if runtime.GOOS == windows {
name = fmt.Sprintf("%s.exe", name)
}
@ -67,6 +69,18 @@ func VirtualEnvCommand(virtualEnvDir string, name string, arg ...string) *exec.C
return exec.Command(cmdPath, arg...)
}
// IsVirtualEnv returns true if the specified directory contains a python binary.
func IsVirtualEnv(dir string) bool {
pyBin := filepath.Join(dir, virtualEnvBinDirName(), "python")
if runtime.GOOS == windows {
pyBin = fmt.Sprintf("%s.exe", pyBin)
}
if info, err := os.Stat(pyBin); err == nil && !info.IsDir() {
return true
}
return false
}
// ActivateVirtualEnv takes an array of environment variables (same format as os.Environ()) and path to
// a virtual environment directory, and returns a new "activated" array with the virtual environment's
// "bin" dir ("Scripts" on Windows) prepended to the `PATH` environment variable and `PYTHONHOME` variable
@ -96,6 +110,79 @@ func ActivateVirtualEnv(environ []string, virtualEnvDir string) []string {
return result
}
// InstallDependencies will create a new virtual environment and install dependencies in the root directory.
func InstallDependencies(root string, showOutput bool, saveProj func(virtualenv string) error) error {
if showOutput {
fmt.Println("Creating virtual environment...")
fmt.Println()
}
// Create the virtual environment by running `python -m venv venv`.
venvDir := filepath.Join(root, "venv")
cmd, err := Command("-m", "venv", venvDir)
if err != nil {
return err
}
if output, err := cmd.CombinedOutput(); err != nil {
if len(output) > 0 {
os.Stdout.Write(output)
fmt.Println()
}
return errors.Wrapf(err, "creating virtual environment at %s", venvDir)
}
// Save project with venv info.
if err := saveProj("venv"); err != nil {
return err
}
if showOutput {
fmt.Println("Finished creating virtual environment")
fmt.Println()
}
// If `requirements.txt` doesn't exist, just exit early.
requirementsPath := filepath.Join(root, "requirements.txt")
if _, err := os.Stat(requirementsPath); os.IsNotExist(err) {
return nil
}
if showOutput {
fmt.Println("Installing dependencies...")
fmt.Println()
}
// Install dependencies by running `pip install -r requirements.txt` using the `pip`
// in the virtual environment.
pipCmd := VirtualEnvCommand(venvDir, "pip", "install", "-r", "requirements.txt")
pipCmd.Dir = root
pipCmd.Env = ActivateVirtualEnv(os.Environ(), venvDir)
if showOutput {
// Show stdout/stderr output.
pipCmd.Stdout = os.Stdout
pipCmd.Stderr = os.Stderr
if err := pipCmd.Run(); err != nil {
return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`")
}
} else {
// Otherwise, only show output if there is an error.
if output, err := pipCmd.CombinedOutput(); err != nil {
if len(output) > 0 {
os.Stdout.Write(output)
fmt.Println()
}
return errors.Wrap(err, "installing dependencies via `pip install -r requirements.txt`")
}
}
if showOutput {
fmt.Println("Finished installing dependencies")
fmt.Println()
}
return nil
}
func virtualEnvBinDirName() string {
if runtime.GOOS == windows {
return "Scripts"

19
sdk/python/python_test.go
View file

@ -25,6 +25,25 @@ import (
"github.com/stretchr/testify/assert"
)
func TestIsVirtualEnv(t *testing.T) {
// Create a new empty test directory.
tempdir, _ := ioutil.TempDir("", "test-env")
defer os.RemoveAll(tempdir)
// Assert the empty test directory is not a virtual environment.
assert.False(t, IsVirtualEnv(tempdir))
// Create and run a python command to create a virtual environment.
venvDir := filepath.Join(tempdir, "venv")
cmd, err := Command("-m", "venv", venvDir)
assert.NoError(t, err)
err = cmd.Run()
assert.NoError(t, err)
// Assert the new venv directory is a virtual environment.
assert.True(t, IsVirtualEnv(venvDir))
}
func TestActivateVirtualEnv(t *testing.T) {
venvName := "venv"
venvDir := filepath.Join(venvName, "bin")

5
tests/integration/config_basic/python_venv/.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
*.pyc
/.pulumi/
/dist/
/*.egg-info
venv/

3
tests/integration/config_basic/python_venv/Pulumi.yaml Normal file
View file

@ -0,0 +1,3 @@
name: config_basic_py
description: A simple Python program that uses configuration.
runtime: python

53
tests/integration/config_basic/python_venv/__main__.py Normal file
View file

@ -0,0 +1,53 @@
# Copyright 2016-2018, Pulumi Corporation. All rights reserved.
import pulumi
# Just test that basic config works.
config = pulumi.Config('config_basic_py')
# This value is plaintext and doesn't require encryption.
value = config.require('aConfigValue')
assert value == 'this value is a Pythonic value'
# This value is a secret and is encrypted using the passphrase `supersecret`.
secret = config.require('bEncryptedSecret')
assert secret == 'this super Pythonic secret is encrypted'
test_data = [
{
'key': 'outer',
'expected_json': '{"inner":"value"}',
'expected_object': { 'inner': 'value' }
},
{
'key': 'names',
'expected_json': '["a","b","c","super secret name"]',
'expected_object': ['a', 'b', 'c', 'super secret name']
},
{
'key': 'servers',
'expected_json': '[{"host":"example","port":80}]',
'expected_object': [{ 'host': 'example', 'port': 80 }]
},
{
'key': 'a',
'expected_json': '{"b":[{"c":true},{"c":false}]}',
'expected_object': { 'b': [{ 'c': True }, { 'c': False }] }
},
{
'key': 'tokens',
'expected_json': '["shh"]',
'expected_object': ['shh']
},
{
'key': 'foo',
'expected_json': '{"bar":"don\'t tell"}',
'expected_object': { 'bar': "don't tell" }
}
]
for test in test_data:
json = config.require(test['key'])
obj = config.require_object(test['key'])
assert json == test['expected_json']
assert obj == test['expected_object']

0
tests/integration/config_basic/python_venv/requirements.txt Normal file
View file

5
tests/integration/dynamic/python_venv/.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
*.pyc
/.pulumi/
/dist/
/*.egg-info
venv/

3
tests/integration/dynamic/python_venv/Pulumi.yaml Normal file
View file

@ -0,0 +1,3 @@
name: dynamic_py
description: A simple Python program that uses dynamic providers.
runtime: python

21
tests/integration/dynamic/python_venv/__main__.py Normal file
View file

@ -0,0 +1,21 @@
# Copyright 2016-2018, Pulumi Corporation. All rights reserved.
import binascii
import os
from pulumi import ComponentResource, export
from pulumi.dynamic import Resource, ResourceProvider, CreateResult
class RandomResourceProvider(ResourceProvider):
def create(self, props):
val = binascii.b2a_hex(os.urandom(15)).decode("ascii")
return CreateResult(val, { "val": val })
class Random(Resource):
val: str
def __init__(self, name, opts = None):
super().__init__(RandomResourceProvider(), name, {"val": ""}, opts)
r = Random("foo")
export("random_id", r.id)
export("random_val", r.val)

0
tests/integration/dynamic/python_venv/requirements.txt Normal file
View file

1
tests/integration/dynamic/python_venv/step1/README.md Normal file
View file

@ -0,0 +1 @@
Intentionally make no changes.

5
tests/integration/empty/python_venv/.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
*.pyc
/.pulumi/
/dist/
/*.egg-info
venv/

3
tests/integration/empty/python_venv/Pulumi.yaml Normal file
View file

@ -0,0 +1,3 @@
name: emptypy
description: An empty Python Pulumi program.
runtime: python

7
tests/integration/empty/python_venv/__main__.py Normal file
View file

@ -0,0 +1,7 @@
# Copyright 2016-2018, Pulumi Corporation. All rights reserved.
def main():
return None
if __name__ == "__main__":
main()

0
tests/integration/empty/python_venv/requirements.txt Normal file
View file

65
tests/integration/integration_test.go
View file

@ -76,6 +76,18 @@ func TestEmptyPython(t *testing.T) {
})
}
// TestEmptyPythonVenv simply tests that we can run an empty Python project using automatic virtual environment support.
func TestEmptyPythonVenv(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: filepath.Join("empty", "python_venv"),
Dependencies: []string{
filepath.Join("..", "..", "sdk", "python", "env", "src"),
},
Quick: true,
UseAutomaticVirtualEnv: true,
})
}
// TestEmptyGo simply tests that we can build and run an empty Go project.
func TestEmptyGo(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
@ -1087,6 +1099,37 @@ func TestConfigBasicPython(t *testing.T) {
})
}
// Tests basic configuration from the perspective of a Pulumi program using automatic virtual environment support.
func TestConfigBasicPythonVenv(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: filepath.Join("config_basic", "python_venv"),
Dependencies: []string{
filepath.Join("..", "..", "sdk", "python", "env", "src"),
},
Quick: true,
Config: map[string]string{
"aConfigValue": "this value is a Pythonic value",
},
Secrets: map[string]string{
"bEncryptedSecret": "this super Pythonic secret is encrypted",
},
OrderedConfig: []integration.ConfigValue{
{Key: "outer.inner", Value: "value", Path: true},
{Key: "names[0]", Value: "a", Path: true},
{Key: "names[1]", Value: "b", Path: true},
{Key: "names[2]", Value: "c", Path: true},
{Key: "names[3]", Value: "super secret name", Path: true, Secret: true},
{Key: "servers[0].port", Value: "80", Path: true},
{Key: "servers[0].host", Value: "example", Path: true},
{Key: "a.b[0].c", Value: "true", Path: true},
{Key: "a.b[1].c", Value: "false", Path: true},
{Key: "tokens[0]", Value: "shh", Path: true, Secret: true},
{Key: "foo.bar", Value: "don't tell", Path: true, Secret: true},
},
UseAutomaticVirtualEnv: true,
})
}
// Tests basic configuration from the perspective of a Pulumi Go program.
func TestConfigBasicGo(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
@ -1446,6 +1489,28 @@ func TestDynamicPython(t *testing.T) {
})
}
// Tests dynamic provider in Python using automatic virtual environment support.
func TestDynamicPythonVenv(t *testing.T) {
var randomVal string
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: filepath.Join("dynamic", "python_venv"),
Dependencies: []string{
filepath.Join("..", "..", "sdk", "python", "env", "src"),
},
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
randomVal = stack.Outputs["random_val"].(string)
},
EditDirs: []integration.EditDir{{
Dir: "step1",
Additive: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
assert.Equal(t, randomVal, stack.Outputs["random_val"].(string))
},
}},
UseAutomaticVirtualEnv: true,
})
}
func TestResourceWithSecretSerialization(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: "secret_outputs",