Add cmd to support policy pack config validation (#4186)

* Add cmd `pulumi policy validate-config` to do policy pack config validation

CHANGELOG.md

@@ -23,6 +23,9 @@ CHANGELOG
 - Remove obsolete .NET serialization attributes.
   [#4190](https://github.com/pulumi/pulumi/pull/4190)
 
+- Add support for validating Policy Pack configuration.
+  [#4179](https://github.com/pulumi/pulumi/pull/4186)
+
 ## 1.13.0 (2020-03-18)
 - Add support for plugin acquisition for Go programs
   [#4060](https://github.com/pulumi/pulumi/pull/4060)

pkg/backend/httpstate/client/client.go

@@ -144,6 +144,12 @@ func publishPolicyPackPublishComplete(orgName, policyPackName string, versionTag
 		"/api/orgs/%s/policypacks/%s/versions/%s/complete", orgName, policyPackName, versionTag)
 }
 
+// getPolicyPackConfigSchemaPath returns the API path to retrieve the policy pack configuration schema.
+func getPolicyPackConfigSchemaPath(orgName, policyPackName string, versionTag string) string {
+	return fmt.Sprintf(
+		"/api/orgs/%s/policypacks/%s/versions/%s/schema", orgName, policyPackName, versionTag)
+}
+
 // getUpdatePath returns the API path to for the given stack with the given components joined with path separators
 // and appended to the update root.
 func getUpdatePath(update UpdateIdentifier, components ...string) string {
@@ -687,6 +693,18 @@ func (pc *Client) ApplyPolicyPack(ctx context.Context, orgName, policyGroup,
 	return nil
 }
 
+// GetPolicyPackSchema gets Policy Pack config schema.
+func (pc *Client) GetPolicyPackSchema(ctx context.Context, orgName,
+	policyPackName, versionTag string) (*apitype.GetPolicyPackConfigSchemaResponse, error) {
+	var resp apitype.GetPolicyPackConfigSchemaResponse
+	err := pc.restCall(ctx, http.MethodGet,
+		getPolicyPackConfigSchemaPath(orgName, policyPackName, versionTag), nil, nil, &resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "Retrieving policy pack config schema failed")
+	}
+	return &resp, nil
+}
+
 // DisablePolicyPack disables a `PolicyPack` to the Pulumi organization. If policyGroup is not empty,
 // it will disable the PolicyPack on the default PolicyGroup.
 func (pc *Client) DisablePolicyPack(ctx context.Context, orgName string, policyGroup string,

pkg/backend/httpstate/policypack.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pulumi/pulumi/pkg/backend"
 	"github.com/pulumi/pulumi/pkg/backend/httpstate/client"
 	"github.com/pulumi/pulumi/pkg/engine"
+	resourceanalyzer "github.com/pulumi/pulumi/pkg/resource/analyzer"
 	"github.com/pulumi/pulumi/sdk/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/go/common/util/contract"
@@ -210,6 +211,18 @@ func (pack *cloudPolicyPack) Enable(ctx context.Context, policyGroup string, op
 	return pack.cl.ApplyPolicyPack(ctx, pack.ref.orgName, policyGroup, string(pack.ref.name), *op.VersionTag, op.Config)
 }
 
+func (pack *cloudPolicyPack) Validate(ctx context.Context, op backend.PolicyPackOperation) error {
+	schema, err := pack.cl.GetPolicyPackSchema(ctx, pack.ref.orgName, string(pack.ref.name), *op.VersionTag)
+	if err != nil {
+		return err
+	}
+	err = resourceanalyzer.ValidatePolicyPackConfig(schema.ConfigSchema, op.Config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (pack *cloudPolicyPack) Disable(ctx context.Context, policyGroup string, op backend.PolicyPackOperation) error {
 	if op.VersionTag == nil {
 		return pack.cl.DisablePolicyPack(ctx, pack.ref.orgName, policyGroup, string(pack.ref.name), "" /* versionTag */)

pkg/backend/policypack.go

@@ -56,6 +56,9 @@ type PolicyPack interface {
 	// empty, it disables it for the default Policy Group.
 	Disable(ctx context.Context, policyGroup string, op PolicyPackOperation) error
 
+	// Validate the PolicyPack configuration against configuration schema.
+	Validate(ctx context.Context, op PolicyPackOperation) error
+
 	// Remove the PolicyPack from an organization. The Policy Pack must be removed from
 	// all Policy Groups before it can be removed.
 	Remove(ctx context.Context, op PolicyPackOperation) error

pkg/cmd/policy.go

@@ -33,6 +33,7 @@ func newPolicyCmd() *cobra.Command {
 	cmd.AddCommand(newPolicyNewCmd())
 	cmd.AddCommand(newPolicyPublishCmd())
 	cmd.AddCommand(newPolicyRmCmd())
+	cmd.AddCommand(newPolicyValidateCmd())
 
 	return cmd
 }

pkg/cmd/policy_validate.go

@@ -0,0 +1,72 @@
+// Copyright 2016-2020, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pulumi/pulumi/pkg/backend"
+	"github.com/pulumi/pulumi/sdk/go/common/util/cmdutil"
+	"github.com/spf13/cobra"
+)
+
+func newPolicyValidateCmd() *cobra.Command {
+	var argConfig string
+
+	var cmd = &cobra.Command{
+		Use:   "validate-config <org-name>/<policy-pack-name> <version>",
+		Args:  cmdutil.ExactArgs(2),
+		Short: "[PREVIEW] Validate a Policy Pack configuration",
+		Long:  "Validate a Policy Pack configuration against the configuration schema of the specified version.",
+		Run: cmdutil.RunFunc(func(cmd *cobra.Command, cliArgs []string) error {
+			// Obtain current PolicyPack, tied to the Pulumi service backend.
+			policyPack, err := requirePolicyPack(cliArgs[0])
+			if err != nil {
+				return err
+			}
+
+			// Get version from cmd argument
+			version := &cliArgs[1]
+
+			// Load the configuration from the user-specified JSON file into config object.
+			var config map[string]*json.RawMessage
+			if argConfig != "" {
+				config, err = loadPolicyConfigFromFile(argConfig)
+				if err != nil {
+					return err
+				}
+			}
+
+			err = policyPack.Validate(commandContext(),
+				backend.PolicyPackOperation{
+					VersionTag: version,
+					Scopes:     cancellationScopes,
+					Config:     config,
+				})
+			if err != nil {
+				return err
+			}
+			fmt.Println("Policy Pack configuration is valid.")
+			return nil
+		}),
+	}
+
+	cmd.Flags().StringVar(&argConfig, "config", "",
+		"The file path for the Policy Pack configuration file")
+	cmd.MarkFlagRequired("config") // nolint: errcheck
+
+	return cmd
+}

pkg/resource/analyzer/config.go

@@ -192,6 +192,38 @@ func validatePolicyConfig(schema plugin.AnalyzerPolicyConfigSchema, config map[s
 	return errors, nil
 }
 
+// ValidatePolicyPackConfig validates a policy pack configuration against the specified config schema.
+func ValidatePolicyPackConfig(schemaMap map[string]apitype.PolicyConfigSchema,
+	config map[string]*json.RawMessage) (err error) {
+	for property, schema := range schemaMap {
+		schemaLoader := gojsonschema.NewGoLoader(schema)
+
+		// If the config for this property is nil, we override it with an empty
+		// json struct to ensure the config is not missing any required properties.
+		propertyConfig := config[property]
+		if propertyConfig == nil {
+			temp := json.RawMessage([]byte(`{}`))
+			propertyConfig = &temp
+		}
+		configLoader := gojsonschema.NewBytesLoader(*propertyConfig)
+		result, err := gojsonschema.Validate(schemaLoader, configLoader)
+		if err != nil {
+			return errors.Wrap(err, "unable to validate schema")
+		}
+
+		// If the result is invalid, we need to gather the errors to return to the user.
+		if !result.Valid() {
+			resultErrs := make([]string, len(result.Errors()))
+			for i, e := range result.Errors() {
+				resultErrs[i] = e.Description()
+			}
+			msg := fmt.Sprintf("policy pack configuration is invalid: %s", strings.Join(resultErrs, ", "))
+			return errors.New(msg)
+		}
+	}
+	return err
+}
+
 func convertSchema(schema plugin.AnalyzerPolicyConfigSchema) plugin.JSONSchema {
 	result := plugin.JSONSchema{}
 	result["type"] = "object"

tests/integration/policy/policy_pack_w_config/configs/invalid-required-prop.json

@@ -0,0 +1,5 @@
+{
+    "s3-no-public-read": {
+        "enforcementLevel": "mandatory"
+    }
+}

tests/integration/policy/policy_pack_w_config/index.ts

@@ -17,6 +17,7 @@ if (!packName) {
                 description: "Prohibits setting the publicRead or publicReadWrite permission on AWS S3 buckets.",
                 enforcementLevel: "mandatory",
                 configSchema: {
+                    required: ["message"],
                     properties: {
                         message: {
                             type: "string",

tests/integration/policy/policy_test.go

@@ -52,6 +52,23 @@ func TestPolicyWithConfig(t *testing.T) {
 	// Enable, Disable and then Delete the Policy Pack.
 	e.RunCommand("pulumi", "policy", "enable", fmt.Sprintf("%s/%s", orgName, policyPackName), "0.0.1")
 
+	// Validate Policy Pack Configuration.
+	e.RunCommand("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName),
+		"--config=configs/valid-config.json", "0.0.1")
+	// Valid config, but no version specified.
+	e.RunCommandExpectError("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName),
+		"--config=configs/config.json")
+	// Invalid configs
+	e.RunCommandExpectError("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName),
+		"--config=configs/invalid-config.json", "0.0.1")
+	// Invalid - missing required property.
+	e.RunCommandExpectError("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName),
+		"--config=configs/invalid-required-prop.json", "0.0.1")
+	// Required config flag not present.
+	e.RunCommandExpectError("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName))
+	e.RunCommandExpectError("pulumi", "policy", "validate-config", fmt.Sprintf("%s/%s", orgName, policyPackName),
+		"--config", "0.0.1")
+
 	// Enable Policy Pack with Configuration.
 	e.RunCommand("pulumi", "policy", "enable", fmt.Sprintf("%s/%s", orgName, policyPackName),
 		"--config=configs/valid-config.json", "0.0.1")