Support for running Python policy packs (#4057)

These changes enable running policy packs written in Python.

CHANGELOG.md

@@ -21,7 +21,10 @@ CHANGELOG
 
 - Fix `pulumi stack ls` on Windows
   [#4094](https://github.com/pulumi/pulumi/pull/4094)
-  
+
+- Add support for running Python policy packs.
+  [#4057](https://github.com/pulumi/pulumi/pull/4057)
+
 ## 1.12.1 (2020-03-11)
 - Fix Kubernetes YAML parsing error in .NET.
   [#4023](https://github.com/pulumi/pulumi/pull/4023)

pkg/resource/plugin/analyzer_plugin.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
@@ -78,12 +79,22 @@ func NewAnalyzer(host Host, ctx *Context, name tokens.QName) (Analyzer, error) {
 	}, nil
 }
 
-const policyAnalyzerName = "policy"
-
 // NewPolicyAnalyzer boots the nodejs analyzer plugin located at `policyPackpath`
 func NewPolicyAnalyzer(
 	host Host, ctx *Context, name tokens.QName, policyPackPath string, opts *PolicyAnalyzerOptions) (Analyzer, error) {
 
+	proj, err := workspace.LoadPolicyPack(filepath.Join(policyPackPath, "PulumiPolicy.yaml"))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to load Pulumi policy project located at %q", policyPackPath)
+	}
+
+	// For historical reasons, the Node.js plugin name is just "policy".
+	// All other languages have the runtime appended, e.g. "policy-<runtime>".
+	policyAnalyzerName := "policy"
+	if !strings.EqualFold(proj.Runtime.Name(), "nodejs") {
+		policyAnalyzerName = fmt.Sprintf("policy-%s", proj.Runtime.Name())
+	}
+
 	// Load the policy-booting analyzer plugin (i.e., `pulumi-analyzer-${policyAnalyzerName}`).
 	_, pluginPath, err := workspace.GetPluginPath(
 		workspace.AnalyzerPlugin, policyAnalyzerName, nil)
@@ -97,7 +108,7 @@ func NewPolicyAnalyzer(
 	}
 
 	// Create the environment variables from the options.
-	env, err := constructEnv(opts)
+	env, err := constructEnv(opts, proj.Runtime.Name())
 	if err != nil {
 		return nil, err
 	}
@@ -600,7 +611,7 @@ func convertDiagnostics(protoDiagnostics []*pulumirpc.AnalyzeDiagnostic) ([]Anal
 // constructEnv creates a slice of key/value pairs to be used as the environment for the policy pack process. Each entry
 // is of the form "key=value". Config is passed as an environment variable (including unecrypted secrets), similar to
 // how config is passed to each language runtime plugin.
-func constructEnv(opts *PolicyAnalyzerOptions) ([]string, error) {
+func constructEnv(opts *PolicyAnalyzerOptions, runtime string) ([]string, error) {
 	env := os.Environ()
 
 	maybeAppendEnv := func(k, v string) {
@@ -616,9 +627,19 @@ func constructEnv(opts *PolicyAnalyzerOptions) ([]string, error) {
 	maybeAppendEnv("PULUMI_CONFIG", config)
 
 	if opts != nil {
-		maybeAppendEnv("PULUMI_NODEJS_PROJECT", opts.Project)
-		maybeAppendEnv("PULUMI_NODEJS_STACK", opts.Stack)
-		maybeAppendEnv("PULUMI_NODEJS_DRY_RUN", fmt.Sprintf("%v", opts.DryRun))
+		// Set both PULUMI_NODEJS_* and PULUMI_* environment variables for Node.js. The Node.js
+		// SDK currently looks for the PULUMI_NODEJS_* variants only, but we'd like to move to
+		// using the more general PULUMI_* variants for all languages to avoid special casing
+		// like this, and setting the PULUMI_* variants for Node.js is the first step.
+		if runtime == "nodejs" {
+			maybeAppendEnv("PULUMI_NODEJS_PROJECT", opts.Project)
+			maybeAppendEnv("PULUMI_NODEJS_STACK", opts.Stack)
+			maybeAppendEnv("PULUMI_NODEJS_DRY_RUN", fmt.Sprintf("%v", opts.DryRun))
+		}
+
+		maybeAppendEnv("PULUMI_PROJECT", opts.Project)
+		maybeAppendEnv("PULUMI_STACK", opts.Stack)
+		maybeAppendEnv("PULUMI_DRY_RUN", fmt.Sprintf("%v", opts.DryRun))
 	}
 
 	return env, nil

scripts/make_release.ps1

@@ -37,6 +37,7 @@ CopyPackage "$Root\sdk\nodejs\bin" "pulumi"
 Copy-Item "$Root\sdk\nodejs\dist\pulumi-resource-pulumi-nodejs.cmd" "$PublishDir\bin"
 Copy-Item "$Root\sdk\python\dist\pulumi-resource-pulumi-python.cmd" "$PublishDir\bin"
 Copy-Item "$Root\sdk\nodejs\dist\pulumi-analyzer-policy.cmd" "$PublishDir\bin"
+Copy-Item "$Root\sdk\python\dist\pulumi-analyzer-policy-python.cmd" "$PublishDir\bin"
 Copy-Item "$Root\sdk\python\cmd\pulumi-language-python-exec" "$PublishDir\bin"
 
 # By default, if the archive already exists, 7zip will just add files to it, so blow away the existing

scripts/make_release.sh

@@ -56,6 +56,7 @@ run_go_build "${ROOT}/sdk/go/pulumi-language-go"
 cp "${ROOT}/sdk/nodejs/dist/pulumi-resource-pulumi-nodejs" "${PUBDIR}/bin/"
 cp "${ROOT}/sdk/python/dist/pulumi-resource-pulumi-python" "${PUBDIR}/bin/"
 cp "${ROOT}/sdk/nodejs/dist/pulumi-analyzer-policy" "${PUBDIR}/bin/"
+cp "${ROOT}/sdk/python/dist/pulumi-analyzer-policy-python" "${PUBDIR}/bin/"
 cp "${ROOT}/sdk/python/cmd/pulumi-language-python-exec" "${PUBDIR}/bin/"
 
 # Copy packages

sdk/python/Makefile

@@ -30,6 +30,7 @@ lint::
 install_package::
 	cp ./cmd/pulumi-language-python-exec "$(PULUMI_BIN)"
 	cp ./dist/pulumi-resource-pulumi-python "$(PULUMI_BIN)"
+	cp ./dist/pulumi-analyzer-policy-python "$(PULUMI_BIN)"
 
 install_plugin::
 	GOBIN=$(PULUMI_BIN) go install \
@@ -47,3 +48,4 @@ dist::
 	go install -ldflags "-X github.com/pulumi/pulumi/sdk/python/pkg/version.Version=${VERSION}" ${LANGHOST_PKG}
 	cp ./cmd/pulumi-language-python-exec "$$(go env GOPATH)"/bin/
 	cp ./dist/pulumi-resource-pulumi-python "$$(go env GOPATH)"/bin/
+	cp ./dist/pulumi-analyzer-policy-python "$$(go env GOPATH)"/bin/

sdk/python/dist/pulumi-analyzer-policy-python

@@ -0,0 +1,2 @@
+#!/bin/sh
+python3 -u -m pulumi.policy $@

sdk/python/dist/pulumi-analyzer-policy-python.cmd

@@ -0,0 +1,5 @@
+@echo off
+setlocal
+REM We use `python` instead of `python3` because Windows Python installers
+REM install only `python.exe` by default.
+@python -u -m pulumi.policy %*

sdk/python/lib/pulumi/__init__.py

@@ -19,7 +19,7 @@ resources.
 """
 
 # Make subpackages available.
-__all__ = ['runtime', 'dynamic']
+__all__ = ['runtime', 'dynamic', 'policy']
 
 # Make all module members inside of this package available as package members.
 from .asset import (

sdk/python/lib/pulumi/policy/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2016-2020, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The Pulumi SDK's policy module. This is meant for internal use only.
+"""

sdk/python/lib/pulumi/policy/__main__.py

@@ -0,0 +1,65 @@
+# Copyright 2016-2020, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import traceback
+import runpy
+
+import pulumi
+import pulumi.runtime
+
+
+def main():
+    if len(sys.argv) != 3:
+        # For whatever reason, sys.stderr.write is not picked up by the engine as a message, but 'print' is. The Python
+        # langhost automatically flushes stdout and stderr on shutdown, so we don't need to do it here - just trust that
+        # Python does the sane thing when printing to stderr.
+        print("usage: python3 -u -m pulumi.policy <engine-address> <program>", file=sys.stderr)
+        sys.exit(1)
+
+    program = sys.argv[2]
+
+    # If any config variables are present, parse and set them, so subsequent accesses are fast.
+    config_env = pulumi.runtime.get_config_env()
+    for k, v in config_env.items():
+        pulumi.runtime.set_config(k, v)
+
+    # Configure the runtime so that the user program hooks up to Pulumi as appropriate.
+    if 'PULUMI_PROJECT' in os.environ and 'PULUMI_STACK' in os.environ and 'PULUMI_DRY_RUN' in os.environ:
+        pulumi.runtime.configure(
+            pulumi.runtime.Settings(
+                project=os.environ["PULUMI_PROJECT"],
+                stack=os.environ["PULUMI_STACK"],
+                dry_run=os.environ["PULUMI_DRY_RUN"] == "true"
+            )
+        )
+
+    successful = False
+
+    try:
+        runpy.run_path(program, run_name="__main__")
+        successful = True
+    except Exception:
+        pulumi.log.error("Program failed with an unhandled exception:")
+        pulumi.log.error(traceback.format_exc())
+    finally:
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+    exit_code = 0 if successful else 1
+    sys.exit(exit_code)
+
+
+main()