From c08714ffb494154ab3fce2da9175760c8119cda9 Mon Sep 17 00:00:00 2001 From: Justin Van Patten Date: Fri, 1 Nov 2019 13:41:27 -0700 Subject: [PATCH] Support lists and maps in config (#3342) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds support for lists and maps in config. We now allow lists/maps (and nested structures) in `Pulumi..yaml` (or `Pulumi..json`; yes, we currently support that). For example: ```yaml config: proj:blah: - a - b - c proj:hello: world proj:outer: inner: value proj:servers: - port: 80 ``` While such structures could be specified in the `.yaml` file manually, we support setting values in maps/lists from the command line. As always, you can specify single values with: ```shell $ pulumi config set hello world ``` Which results in the following YAML: ```yaml proj:hello world ``` And single value secrets via: ```shell $ pulumi config set --secret token shhh ``` Which results in the following YAML: ```yaml proj:token: secure: v1:VZAhuroR69FkEPTk:isKafsoZVMWA9pQayGzbWNynww== ``` Values in a list can be set from the command line using the new `--path` flag, which indicates the config key contains a path to a property in a map or list: ```shell $ pulumi config set --path names[0] a $ pulumi config set --path names[1] b $ pulumi config set --path names[2] c ``` Which results in: ```yaml proj:names - a - b - c ``` Values can be obtained similarly: ```shell $ pulumi config get --path names[1] b ``` Or setting values in a map: ```shell $ pulumi config set --path outer.inner value ``` Which results in: ```yaml proj:outer: inner: value ``` Of course, setting values in nested structures is supported: ```shell $ pulumi config set --path servers[0].port 80 ``` Which results in: ```yaml proj:servers: - port: 80 ``` If you want to include a period in the name of a property, it can be specified as: ``` $ pulumi config set --path 'nested["foo.bar"]' baz ``` Which results in: ```yaml proj:nested: foo.bar: baz ``` Examples of valid paths: - root - root.nested - 'root["nested"]' - root.double.nest - 'root["double"].nest' - 'root["double"]["nest"]' - root.array[0] - root.array[100] - root.array[0].nested - root.array[0][1].nested - root.nested.array[0].double[1] - 'root["key with \"escaped\" quotes"]' - 'root["key with a ."]' - '["root key with \"escaped\" quotes"].nested' - '["root key with a ."][100]' Note: paths that contain quotes can be surrounded by single quotes. When setting values with `--path`, if the value is `"false"` or `"true"`, it will be saved as the boolean value, and if it is convertible to an integer, it will be saved as an integer. Secure values are supported in lists/maps as well: ```shell $ pulumi config set --path --secret tokens[0] shh ``` Will result in: ```yaml proj:tokens: - secure: v1:wpZRCe36sFg1RxwG:WzPeQrCn4n+m4Ks8ps15MxvFXg== ``` Note: maps of length 1 with a key of “secure” and string value are reserved for storing secret values. Attempting to create such a value manually will result in an error: ```shell $ pulumi config set --path parent.secure foo error: "secure" key in maps of length 1 are reserved ``` **Accessing config values from the command line with JSON** ```shell $ pulumi config --json ``` Will output: ```json { "proj:hello": { "value": "world", "secret": false, "object": false }, "proj:names": { "value": "[\"a\",\"b\",\"c\"]", "secret": false, "object": true, "objectValue": [ "a", "b", "c" ] }, "proj:nested": { "value": "{\"foo.bar\":\"baz\"}", "secret": false, "object": true, "objectValue": { "foo.bar": "baz" } }, "proj:outer": { "value": "{\"inner\":\"value\"}", "secret": false, "object": true, "objectValue": { "inner": "value" } }, "proj:servers": { "value": "[{\"port\":80}]", "secret": false, "object": true, "objectValue": [ { "port": 80 } ] }, "proj:token": { "secret": true, "object": false }, "proj:tokens": { "secret": true, "object": true } } ``` If the value is a map or list, `"object"` will be `true`. `"value"` will contain the object as serialized JSON and a new `"objectValue"` property will be available containing the value of the object. If the object contains any secret values, `"secret"` will be `true`, and just like with scalar values, the value will not be outputted unless `--show-secrets` is specified. **Accessing config values from Pulumi programs** Map/list values are available to Pulumi programs as serialized JSON, so the existing `getObject`/`requireObject`/`getSecretObject`/`requireSecretObject` functions can be used to retrieve such values, e.g.: ```typescript import * as pulumi from "@pulumi/pulumi"; interface Server { port: number; } const config = new pulumi.Config(); const names = config.requireObject("names"); for (const n of names) { console.log(n); } const servers = config.requireObject("servers"); for (const s of servers) { console.log(s.port); } ``` --- CHANGELOG.md | 3 + cmd/config.go | 79 +- cmd/history.go | 9 + cmd/new.go | 13 +- cmd/new_test.go | 273 +++++ cmd/preview.go | 6 +- cmd/up.go | 11 +- cmd/util.go | 4 +- pkg/backend/httpstate/backend.go | 14 +- pkg/backend/httpstate/client/client.go | 15 +- pkg/engine/events.go | 4 +- pkg/resource/asset.go | 9 +- pkg/resource/config/crypt.go | 30 + pkg/resource/config/map.go | 417 +++++++ pkg/resource/config/map_test.go | 1023 ++++++++++++++++- pkg/resource/config/value.go | 239 +++- pkg/resource/config/value_test.go | 175 +++ pkg/testing/integration/program.go | 31 +- pkg/workspace/paths.go | 2 +- .../config_basic/dotnet/Program.cs | 137 ++- tests/integration/config_basic/go/main.go | 50 +- .../integration/config_basic/nodejs/index.ts | 48 +- .../config_basic/python/__main__.py | 39 + tests/integration/integration_test.go | 353 +++++- 24 files changed, 2884 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed77fc701..268545e39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ CHANGELOG - `pulumi stack` now renders the stack as a tree view. [#3430](https://github.com/pulumi/pulumi/pull/3430) +- Support for lists and maps in config. + [#3342](https://github.com/pulumi/pulumi/pull/3342) + ## 1.4.0 (2019-10-24) - `FileAsset` in the Python SDK now accepts anything implementing `os.PathLike` in addition to `str`. diff --git a/cmd/config.go b/cmd/config.go index e2ee84d1e..537638651 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -86,11 +86,18 @@ func newConfigCmd() *cobra.Command { func newConfigGetCmd(stack *string) *cobra.Command { var jsonOut bool + var path bool getCmd := &cobra.Command{ Use: "get ", Short: "Get a single configuration value", - Args: cmdutil.SpecificArgs([]string{"key"}), + Long: "Get a single configuration value.\n\n" + + "The `--path` flag can be used to get a value inside a map or list:\n\n" + + " - `pulumi config get --path outer.inner` will get the value of the `inner` key, " + + "if the value of `outer` is a map `inner: value`.\n" + + " - `pulumi config get --path names[0]` will get the value of the first item, " + + "if the value of `names` is a list.", + Args: cmdutil.SpecificArgs([]string{"key"}), Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error { opts := display.Options{ Color: cmdutil.GetGlobalColorization(), @@ -106,21 +113,32 @@ func newConfigGetCmd(stack *string) *cobra.Command { return errors.Wrap(err, "invalid configuration key") } - return getConfig(s, key, jsonOut) + return getConfig(s, key, path, jsonOut) }), } getCmd.Flags().BoolVarP( &jsonOut, "json", "j", false, "Emit output as JSON") + getCmd.PersistentFlags().BoolVar( + &path, "path", false, + "The key contains a path to a property in a map or list to get") return getCmd } func newConfigRmCmd(stack *string) *cobra.Command { + var path bool + rmCmd := &cobra.Command{ Use: "rm ", Short: "Remove configuration value", - Args: cmdutil.SpecificArgs([]string{"key"}), + Long: "Remove configuration value.\n\n" + + "The `--path` flag can be used to remove a value inside a map or list:\n\n" + + " - `pulumi config rm --path outer.inner` will remove the `inner` key, " + + "if the value of `outer` is a map `inner: value`.\n" + + " - `pulumi config rm --path names[0]` will remove the first item, " + + "if the value of `names` is a list.", + Args: cmdutil.SpecificArgs([]string{"key"}), Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error { opts := display.Options{ Color: cmdutil.GetGlobalColorization(), @@ -141,13 +159,17 @@ func newConfigRmCmd(stack *string) *cobra.Command { return err } - if ps.Config != nil { - delete(ps.Config, key) + err = ps.Config.Remove(key, path) + if err != nil { + return err } return saveProjectStack(s, ps) }), } + rmCmd.PersistentFlags().BoolVar( + &path, "path", false, + "The key contains a path to a property in a map or list to remove") return rmCmd } @@ -226,13 +248,19 @@ func newConfigRefreshCmd(stack *string) *cobra.Command { func newConfigSetCmd(stack *string) *cobra.Command { var plaintext bool var secret bool + var path bool setCmd := &cobra.Command{ Use: "set [value]", Short: "Set configuration value", Long: "Configuration values can be accessed when a stack is being deployed and used to configure behavior. \n" + "If a value is not present on the command line, pulumi will prompt for the value. Multi-line values\n" + - "may be set by piping a file to standard in.", + "may be set by piping a file to standard in.\n\n" + + "The `--path` flag can be used to set a value inside a map or list:\n\n" + + " - `pulumi config set --path outer.inner value` " + + "will set the value of `outer` to a map `inner: value`.\n" + + " - `pulumi config set --path names[0] a` " + + "will set the value to a list with the first item `a`.", Args: cmdutil.RangeArgs(1, 2), Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error { opts := display.Options{ @@ -301,12 +329,18 @@ func newConfigSetCmd(stack *string) *cobra.Command { return err } - ps.Config[key] = v + err = ps.Config.Set(key, v, path) + if err != nil { + return err + } return saveProjectStack(s, ps) }), } + setCmd.PersistentFlags().BoolVar( + &path, "path", false, + "The key contains a path to a property in a map or list to set") setCmd.PersistentFlags().BoolVar( &plaintext, "plaintext", false, "Save the value as plaintext (unencrypted)") @@ -376,8 +410,10 @@ func prettyKeyForProject(k config.Key, proj *workspace.Project) string { // structure in the future, we should not change existing fields. type configValueJSON struct { // When the value is encrypted and --show-secrets was not passed, the value will not be set. - Value *string `json:"value,omitempty"` - Secret bool `json:"secret"` + // If the value is an object, ObjectValue will be set. + Value *string `json:"value,omitempty"` + ObjectValue interface{} `json:"objectValue,omitempty"` + Secret bool `json:"secret"` } func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error { @@ -419,11 +455,20 @@ func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error { } entry.Value = &decrypted + if cfg[key].Object() { + var obj interface{} + if err := json.Unmarshal([]byte(decrypted), &obj); err != nil { + return err + } + entry.ObjectValue = obj + } + // If the value was a secret value and we aren't showing secrets, then the above would have set value // to "[secret]" which is reasonable when printing for human display, but for our JSON output, we'd rather // just elide the value. if cfg[key].Secure() && !showSecrets { entry.Value = nil + entry.ObjectValue = nil } configValues[key.String()] = entry @@ -453,7 +498,7 @@ func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error { return nil } -func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error { +func getConfig(stack backend.Stack, key config.Key, path, jsonOut bool) error { ps, err := loadProjectStack(stack) if err != nil { return err @@ -461,7 +506,11 @@ func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error { cfg := ps.Config - if v, ok := cfg[key]; ok { + v, ok, err := cfg.Get(key, path) + if err != nil { + return err + } + if ok { var d config.Decrypter if v.Secure() { var err error @@ -482,6 +531,14 @@ func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error { Secret: v.Secure(), } + if v.Object() { + var obj interface{} + if err := json.Unmarshal([]byte(raw), &obj); err != nil { + return err + } + value.ObjectValue = obj + } + out, err := json.MarshalIndent(value, "", " ") if err != nil { return err diff --git a/cmd/history.go b/cmd/history.go index d92530274..33a0e8db6 100644 --- a/cmd/history.go +++ b/cmd/history.go @@ -15,6 +15,7 @@ package cmd import ( + "encoding/json" "fmt" "sort" "strings" @@ -123,6 +124,14 @@ func displayUpdatesJSON(updates []backend.UpdateInfo, decrypter config.Decrypter value, err := v.Value(decrypter) contract.AssertNoError(err) configValue.Value = makeStringRef(value) + + if v.Object() { + var obj interface{} + if err := json.Unmarshal([]byte(value), &obj); err != nil { + return err + } + configValue.ObjectValue = obj + } } info.Config[k.String()] = configValue diff --git a/cmd/new.go b/cmd/new.go index 4600610be..8156e37da 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -49,6 +49,7 @@ type promptForValueFunc func(yes bool, valueType string, defaultValue string, se type newArgs struct { configArray []string + configPath bool description string dir string force bool @@ -253,7 +254,8 @@ func runNew(args newArgs) error { // Prompt for config values (if needed) and save. if !args.generateOnly { - if err = handleConfig(s, args.templateNameOrURL, template, args.configArray, args.yes, opts); err != nil { + err = handleConfig(s, args.templateNameOrURL, template, args.configArray, args.yes, args.configPath, opts) + if err != nil { return err } } @@ -384,6 +386,9 @@ func newNewCmd() *cobra.Command { cmd.PersistentFlags().StringArrayVarP( &args.configArray, "config", "c", []string{}, "Config to save") + cmd.PersistentFlags().BoolVar( + &args.configPath, "config-path", false, + "Config keys contain a path to a property in a map or list to set") cmd.PersistentFlags().StringVarP( &args.description, "description", "d", "", "The project description; if not specified, a prompt will request it") @@ -724,7 +729,7 @@ func chooseTemplate(templates []workspace.Template, opts display.Options) (works // These are passed as `-c aws:region=us-east-1 -c foo:bar=blah` and end up // in configArray as ["aws:region=us-east-1", "foo:bar=blah"]. // This function converts the array into a config.Map. -func parseConfig(configArray []string) (config.Map, error) { +func parseConfig(configArray []string, path bool) (config.Map, error) { configMap := make(config.Map) for _, c := range configArray { kvp := strings.SplitN(c, "=", 2) @@ -739,7 +744,9 @@ func parseConfig(configArray []string) (config.Map, error) { value = config.NewValue(kvp[1]) } - configMap[key] = value + if err = configMap.Set(key, value, path); err != nil { + return nil, err + } } return configMap, nil } diff --git a/cmd/new_test.go b/cmd/new_test.go index b2e189287..d952cd8e9 100644 --- a/cmd/new_test.go +++ b/cmd/new_test.go @@ -23,6 +23,7 @@ import ( "github.com/pulumi/pulumi/pkg/backend" "github.com/pulumi/pulumi/pkg/backend/display" + "github.com/pulumi/pulumi/pkg/resource/config" "github.com/pulumi/pulumi/pkg/workspace" "github.com/stretchr/testify/assert" ) @@ -448,6 +449,278 @@ func TestInvalidTemplateName(t *testing.T) { }) } +func TestParseConfigSuccess(t *testing.T) { + tests := []struct { + Array []string + Path bool + Expected config.Map + }{ + { + Array: []string{}, + Expected: config.Map{}, + }, + { + Array: []string{"my:testKey"}, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue(""), + }, + }, + { + Array: []string{"my:testKey="}, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue(""), + }, + }, + { + Array: []string{"my:testKey=testValue"}, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("testValue"), + }, + }, + { + Array: []string{"my:testKey=test=Value"}, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("test=Value"), + }, + }, + { + Array: []string{ + "my:testKey=testValue", + "my:testKey=rewritten", + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("rewritten"), + }, + }, + { + Array: []string{ + "my:testKey=testValue", + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + "my:test.Key=testValue", + }, + Expected: config.Map{ + config.MustMakeKey("my", "test.Key"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + "my:testKey=testValue", + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + "my:0=testValue", + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "0"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + "my:true=testValue", + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "true"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + `my:["test.Key"]=testValue`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "test.Key"): config.NewValue("testValue"), + }, + }, + { + Array: []string{ + `my:outer.inner=value`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "outer"): config.NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Array: []string{ + `my:outer.inner.nested=value`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "outer"): config.NewObjectValue(`{"inner":{"nested":"value"}}`), + }, + }, + { + Array: []string{ + `my:name[0]=value`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "name"): config.NewObjectValue(`["value"]`), + }, + }, + { + Array: []string{ + `my:name[0][0]=value`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "name"): config.NewObjectValue(`[["value"]]`), + }, + }, + { + Array: []string{ + `my:servers[0].name=foo`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "servers"): config.NewObjectValue(`[{"name":"foo"}]`), + }, + }, + { + Array: []string{ + `my:testKey=false`, + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("false"), + }, + }, + { + Array: []string{ + `my:testKey=true`, + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("true"), + }, + }, + { + Array: []string{ + `my:testKey=10`, + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("10"), + }, + }, + { + Array: []string{ + `my:testKey=-1`, + }, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewValue("-1"), + }, + }, + { + Array: []string{ + `my:testKey[0]=false`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[false]`), + }, + }, + { + Array: []string{ + `my:testKey[0]=true`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[true]`), + }, + }, + { + Array: []string{ + `my:testKey[0]=10`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[10]`), + }, + }, + { + Array: []string{ + `my:testKey[0]=-1`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[-1]`), + }, + }, + { + Array: []string{ + `my:names[0]=a`, + `my:names[1]=b`, + `my:names[2]=c`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "names"): config.NewObjectValue(`["a","b","c"]`), + }, + }, + { + Array: []string{ + `my:names[0]=a`, + `my:names[1]=b`, + `my:names[2]=c`, + `my:names[0]=rewritten`, + }, + Path: true, + Expected: config.Map{ + config.MustMakeKey("my", "names"): config.NewObjectValue(`["rewritten","b","c"]`), + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + actual, err := parseConfig(test.Array, test.Path) + assert.NoError(t, err) + assert.Equal(t, test.Expected, actual) + }) + } +} + +func TestSetFail(t *testing.T) { + tests := []struct { + Array []string + Expected config.Map + }{ + { + Array: []string{`my:[""]=value`}, + }, + { + Array: []string{"my:[0]=value"}, + }, + { + Array: []string{`my:name[-1]=value`}, + }, + { + Array: []string{`my:name[1]=value`}, + }, + { + Array: []string{`my:key.secure=value`}, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + _, err := parseConfig(test.Array, true /*path*/) + assert.Error(t, err) + }) + } +} + const projectName = "test_project" const stackName = "test_stack" diff --git a/cmd/preview.go b/cmd/preview.go index 987284c75..a9b3ba531 100644 --- a/cmd/preview.go +++ b/cmd/preview.go @@ -31,6 +31,7 @@ func newPreviewCmd() *cobra.Command { var message string var stack string var configArray []string + var configPath bool // Flags for engine.UpdateOptions. var policyPackPaths []string @@ -95,7 +96,7 @@ func newPreviewCmd() *cobra.Command { } // Save any config values passed via flags. - if err := parseAndSaveConfigArray(s, configArray); err != nil { + if err := parseAndSaveConfigArray(s, configArray, configPath); err != nil { return result.FromError(err) } @@ -155,6 +156,9 @@ func newPreviewCmd() *cobra.Command { cmd.PersistentFlags().StringArrayVarP( &configArray, "config", "c", []string{}, "Config to use during the preview") + cmd.PersistentFlags().BoolVar( + &configPath, "config-path", false, + "Config keys contain a path to a property in a map or list to set") cmd.PersistentFlags().StringVarP( &message, "message", "m", "", diff --git a/cmd/up.go b/cmd/up.go index 1c3760d70..99deb4b79 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -51,6 +51,7 @@ func newUpCmd() *cobra.Command { var message string var stack string var configArray []string + var path bool // Flags for engine.UpdateOptions. var policyPackPaths []string @@ -78,7 +79,7 @@ func newUpCmd() *cobra.Command { } // Save any config values passed via flags. - if err := parseAndSaveConfigArray(s, configArray); err != nil { + if err := parseAndSaveConfigArray(s, configArray, path); err != nil { return result.FromError(err) } @@ -253,7 +254,7 @@ func newUpCmd() *cobra.Command { } // Prompt for config values (if needed) and save. - if err = handleConfig(s, templateNameOrURL, template, configArray, yes, opts.Display); err != nil { + if err = handleConfig(s, templateNameOrURL, template, configArray, yes, path, opts.Display); err != nil { return result.FromError(err) } @@ -379,6 +380,9 @@ func newUpCmd() *cobra.Command { cmd.PersistentFlags().StringArrayVarP( &configArray, "config", "c", []string{}, "Config to use during the update") + cmd.PersistentFlags().BoolVar( + &path, "config-path", false, + "Config keys contain a path to a property in a map or list to set") cmd.PersistentFlags().StringVar( &secretsProvider, "secrets-provider", "default", "The type of the provider that should be used to encrypt and "+ "decrypt secrets (possible choices: default, passphrase, awskms, azurekeyvault, gcpkms, hashivault). Only"+ @@ -454,6 +458,7 @@ func handleConfig( template workspace.Template, configArray []string, yes bool, + path bool, opts display.Options) error { // Get the existing config. stackConfig will be nil if there wasn't a previous deployment. @@ -480,7 +485,7 @@ func handleConfig( // the stack's `pulumi:template` config value. } else { // Get config values passed on the command line. - commandLineConfig, parseErr := parseConfig(configArray) + commandLineConfig, parseErr := parseConfig(configArray, path) if parseErr != nil { return parseErr } diff --git a/cmd/util.go b/cmd/util.go index 0707546e6..184142e58 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -315,11 +315,11 @@ func chooseStack( // parseAndSaveConfigArray parses the config array and saves it as a config for // the provided stack. -func parseAndSaveConfigArray(s backend.Stack, configArray []string) error { +func parseAndSaveConfigArray(s backend.Stack, configArray []string, path bool) error { if len(configArray) == 0 { return nil } - commandLineConfig, err := parseConfig(configArray) + commandLineConfig, err := parseConfig(configArray, path) if err != nil { return err } diff --git a/pkg/backend/httpstate/backend.go b/pkg/backend/httpstate/backend.go index 7e04ec6c2..f046631ac 100644 --- a/pkg/backend/httpstate/backend.go +++ b/pkg/backend/httpstate/backend.go @@ -1007,10 +1007,18 @@ func convertConfig(apiConfig map[string]apitype.ConfigValue) (config.Map, error) if err != nil { return nil, err } - if rawV.Secret { - c[k] = config.NewSecureValue(rawV.String) + if rawV.Object { + if rawV.Secret { + c[k] = config.NewSecureObjectValue(rawV.String) + } else { + c[k] = config.NewObjectValue(rawV.String) + } } else { - c[k] = config.NewValue(rawV.String) + if rawV.Secret { + c[k] = config.NewSecureValue(rawV.String) + } else { + c[k] = config.NewValue(rawV.String) + } } } return c, nil diff --git a/pkg/backend/httpstate/client/client.go b/pkg/backend/httpstate/client/client.go index 4aea2b4f7..2e86e9c2b 100644 --- a/pkg/backend/httpstate/client/client.go +++ b/pkg/backend/httpstate/client/client.go @@ -224,10 +224,18 @@ func (pc *Client) GetLatestConfiguration(ctx context.Context, stackID StackIdent if err != nil { return nil, err } - if v.Secret { - cfg[newKey] = config.NewSecureValue(v.String) + if v.Object { + if v.Secret { + cfg[newKey] = config.NewSecureObjectValue(v.String) + } else { + cfg[newKey] = config.NewObjectValue(v.String) + } } else { - cfg[newKey] = config.NewValue(v.String) + if v.Secret { + cfg[newKey] = config.NewSecureValue(v.String) + } else { + cfg[newKey] = config.NewValue(v.String) + } } } @@ -387,6 +395,7 @@ func (pc *Client) CreateUpdate( wireConfig[k.String()] = apitype.ConfigValue{ String: v, Secret: cv.Secure(), + Object: cv.Object(), } } diff --git a/pkg/engine/events.go b/pkg/engine/events.go index 92fa11580..de359b2cf 100644 --- a/pkg/engine/events.go +++ b/pkg/engine/events.go @@ -175,14 +175,14 @@ func makeEventEmitter(events chan<- Event, update UpdateInfo) (eventEmitter, err continue } - secret, err := v.Value(target.Decrypter) + secureValues, err := v.SecureValues(target.Decrypter) if err != nil { return eventEmitter{}, DecryptError{ Key: k, Err: err, } } - secrets = append(secrets, secret) + secrets = append(secrets, secureValues...) } } diff --git a/pkg/resource/asset.go b/pkg/resource/asset.go index edbb26e80..9dece11fc 100644 --- a/pkg/resource/asset.go +++ b/pkg/resource/asset.go @@ -38,7 +38,12 @@ import ( "github.com/pulumi/pulumi/pkg/util/contract" "github.com/pulumi/pulumi/pkg/util/httputil" - "github.com/pulumi/pulumi/pkg/workspace" +) + +const ( + // BookkeepingDir is the name of our bookkeeping folder, we store state here (like .git for git). + // Copied from workspace.BookkeepingDir to break import cycle. + BookkeepingDir = ".pulumi" ) // Asset is a serialized asset reference. It is a union: thus, only one of its fields will be non-nil. Several helper @@ -828,7 +833,7 @@ func (a *Archive) readPath() (ArchiveReader, error) { // If this is a .pulumi directory, we will skip this by default. // TODO[pulumi/pulumi#122]: when we support .pulumiignore, this will be customizable. - if f.Name() == workspace.BookkeepingDir { + if f.Name() == BookkeepingDir { if f.IsDir() { return filepath.SkipDir } diff --git a/pkg/resource/config/crypt.go b/pkg/resource/config/crypt.go index edbda6763..cdbba4966 100644 --- a/pkg/resource/config/crypt.go +++ b/pkg/resource/config/crypt.go @@ -58,6 +58,36 @@ func (nopCrypter) EncryptValue(plaintext string) (string, error) { return plaintext, nil } +// TrackingDecrypter is a Decrypter that keeps track if decrypted values, which +// can be retrieved via SecureValues(). +type TrackingDecrypter interface { + Decrypter + SecureValues() []string +} + +// NewTrackingDecrypter returns a Decrypter that keeps track of decrypted values. +func NewTrackingDecrypter(decrypter Decrypter) TrackingDecrypter { + return &trackingDecrypter{decrypter: decrypter} +} + +type trackingDecrypter struct { + decrypter Decrypter + secureValues []string +} + +func (t *trackingDecrypter) DecryptValue(ciphertext string) (string, error) { + v, err := t.decrypter.DecryptValue(ciphertext) + if err != nil { + return "", err + } + t.secureValues = append(t.secureValues, v) + return v, nil +} + +func (t *trackingDecrypter) SecureValues() []string { + return t.secureValues +} + // NewBlindingDecrypter returns a Decrypter that instead of decrypting data, just returns "[secret]", it can // be used when you want to display configuration information to a user but don't want to prompt for a password // so secrets will not be decrypted. diff --git a/pkg/resource/config/map.go b/pkg/resource/config/map.go index 891399082..b2068884f 100644 --- a/pkg/resource/config/map.go +++ b/pkg/resource/config/map.go @@ -16,10 +16,18 @@ package config import ( "encoding/json" + "fmt" + "strconv" + + "github.com/pulumi/pulumi/pkg/util/contract" "github.com/pkg/errors" + + "github.com/pulumi/pulumi/pkg/resource" ) +var errSecureKeyReserved = errors.New(`"secure" key in maps of length 1 are reserved`) + // Map is a bag of config stored in the settings file. type Map map[Key]Value @@ -47,6 +55,265 @@ func (m Map) HasSecureValue() bool { return false } +// Get gets the value for a given key. If path is true, the key's name portion is treated as a path. +func (m Map) Get(k Key, path bool) (Value, bool, error) { + // If the key isn't a path, go ahead and lookup the value. + if !path { + v, ok := m[k] + return v, ok, nil + } + + // Otherwise, parse the path and get the new config key. + p, configKey, err := parseKeyPath(k) + if err != nil { + return Value{}, false, err + } + + // If we only have a single path segment, go ahead and lookup the value. + if len(p) == 1 { + v, ok := m[configKey] + return v, ok, nil + } + + // Otherwise, lookup the current root value and save it into a temporary map. + root := make(map[string]interface{}) + if val, ok := m[configKey]; ok { + obj, err := val.ToObject() + if err != nil { + return Value{}, false, err + } + root[configKey.Name()] = obj + } + + // Get the value within the object. + _, v, ok := getValueForPath(root, p) + if !ok { + return Value{}, false, nil + } + + // If the value is a secure value, return it as one. + if is, s := isSecureValue(v); is { + return NewSecureValue(s), true, nil + } + + // If it's a simple type, return it as a regular value. + switch t := v.(type) { + case string: + return NewValue(t), true, nil + case bool, int, uint, int32, uint32, int64, uint64, float32, float64: + return NewValue(fmt.Sprintf("%v", v)), true, nil + } + + // Otherwise, return it as an object value. + json, err := json.Marshal(v) + if err != nil { + return Value{}, false, err + } + if hasSecureValue(v) { + return NewSecureObjectValue(string(json)), true, nil + } + return NewObjectValue(string(json)), true, nil +} + +// Remove removes the value for a given key. If path is true, the key's name portion is treated as a path. +func (m Map) Remove(k Key, path bool) error { + // If the key isn't a path, go ahead and delete it and return. + if !path { + delete(m, k) + return nil + } + + // Parse the path. + p, err := resource.ParsePropertyPath(k.Name()) + if err != nil { + return errors.Wrap(err, "invalid config key path") + } + if len(p) == 0 { + return nil + } + firstKey, ok := p[0].(string) + if !ok || firstKey == "" { + return nil + } + configKey := MustMakeKey(k.Namespace(), firstKey) + + // If we only have a single path segment, delete the key and return. + if len(p) == 1 { + delete(m, configKey) + return nil + } + + // Otherwise, lookup the current root value and save it into a temporary map. + root := make(map[string]interface{}) + if val, ok := m[configKey]; ok { + obj, err := val.ToObject() + if err != nil { + return err + } + root[configKey.Name()] = obj + } + + // Get the value within the object up to the second-to-last path segment. + // If not found, exit early. + parent, dest, ok := getValueForPath(root, p[:len(p)-1]) + if !ok { + return nil + } + + // Remove the last path segment. + key := p[len(p)-1] + switch t := dest.(type) { + case []interface{}: + index, ok := key.(int) + if !ok || index < 0 || index >= len(t) { + return nil + } + t = append(t[:index], t[index+1:]...) + // Since we changed the array, we need to update the parent. + if parent != nil { + pkey := p[len(p)-2] + if _, err := setValue(parent, pkey, t, nil, nil); err != nil { + return err + } + } + case map[string]interface{}: + k, ok := key.(string) + if !ok { + return nil + } + delete(t, k) + + // Secure values are reserved, so return an error when attempting to add one. + if isSecure, _ := isSecureValue(t); isSecure { + return errSecureKeyReserved + } + } + + // Now, marshal then unmarshal the value, which will handle detecting + // whether it's a secure object or not. + jsonBytes, err := json.Marshal(root[configKey.Name()]) + if err != nil { + return err + } + var v Value + if err = json.Unmarshal(jsonBytes, &v); err != nil { + return err + } + + m[configKey] = v + return nil +} + +// Set sets the value for a given key. If path is true, the key's name portion is treated as a path. +func (m Map) Set(k Key, v Value, path bool) error { + // If the key isn't a path, go ahead and set the value and return. + if !path { + m[k] = v + return nil + } + + // Otherwise, parse the path and get the new config key. + p, configKey, err := parseKeyPath(k) + if err != nil { + return err + } + + // If we only have a single path segment, set the value and return. + if len(p) == 1 { + m[configKey] = v + return nil + } + + // Otherwise, lookup the current value and save it into a temporary map. + root := make(map[string]interface{}) + if val, ok := m[configKey]; ok { + obj, err := val.ToObject() + if err != nil { + return err + } + + // If obj is a string, set it to nil, which allows overwriting the existing + // top-level string value in the first iteration of the loop below. + if _, ok := obj.(string); ok { + obj = nil + } + + root[configKey.Name()] = obj + } + + // Now, loop through the path segments, and walk the object tree. + // If the value for a given segment is nil, create a new array/map. + // The root map is the initial cursor value, and parent is nil. + var parent interface{} + var parentKey interface{} + var cursor interface{} + var cursorKey interface{} + cursor = root + cursorKey = p[0] + for _, pkey := range p[1:] { + pvalue, err := getValue(cursor, cursorKey) + if err != nil { + return err + } + + // If the value is nil, create a new array/map. + // Otherwise, return an error due to the type mismatch. + var newValue interface{} + switch pkey.(type) { + case int: + if pvalue == nil { + newValue = make([]interface{}, 0) + } else if _, ok := pvalue.([]interface{}); !ok { + return errors.Errorf("an array was expected for index %v", pkey) + } + case string: + if pvalue == nil { + newValue = make(map[string]interface{}) + } else if _, ok := pvalue.(map[string]interface{}); !ok { + return errors.Errorf("a map was expected for key %q", pkey) + } + default: + contract.Failf("unexpected path type") + } + if newValue != nil { + pvalue = newValue + cursor, err = setValue(cursor, cursorKey, pvalue, parent, parentKey) + if err != nil { + return err + } + } + + parent = cursor + parentKey = cursorKey + cursor = pvalue + cursorKey = pkey + } + + // Adjust the value (e.g. convert "true"/"false" to booleans and integers to ints) and set it. + adjustedValue := adjustObjectValue(v, path) + if _, err = setValue(cursor, cursorKey, adjustedValue, parent, parentKey); err != nil { + return err + } + + // Secure values are reserved, so return an error when attempting to add one. + if isSecure, _ := isSecureValue(cursor); isSecure { + return errSecureKeyReserved + } + + // Serialize the updated object as JSON, and save it in the config map. + json, err := json.Marshal(root[configKey.Name()]) + if err != nil { + return err + } + if v.Secure() { + m[configKey] = NewSecureObjectValue(string(json)) + } else { + m[configKey] = NewObjectValue(string(json)) + } + + return nil +} + func (m Map) MarshalJSON() ([]byte, error) { rawMap := make(map[string]Value, len(m)) for k, v := range m { @@ -104,3 +371,153 @@ func (m *Map) UnmarshalYAML(unmarshal func(interface{}) error) error { *m = newMap return nil } + +// parseKeyPath returns the property paths in the key and a new config key with the first +// path segment as the name. +func parseKeyPath(k Key) (resource.PropertyPath, Key, error) { + // Parse the path, which will be in the name portion of the key. + p, err := resource.ParsePropertyPath(k.Name()) + if err != nil { + return nil, Key{}, errors.Wrap(err, "invalid config key path") + } + if len(p) == 0 { + return nil, Key{}, errors.New("empty config key path") + } + + // Create a new key that has the first path segment as the name. + firstKey, ok := p[0].(string) + if !ok { + return nil, Key{}, errors.New("first path segement of config key must be a string") + } + if firstKey == "" { + return nil, Key{}, errors.New("config key is empty") + } + + configKey := MustMakeKey(k.Namespace(), firstKey) + + return p, configKey, nil +} + +// getValueForPath returns the parent, value, and true if the value is found in source given the path segments in p. +func getValueForPath(source interface{}, p resource.PropertyPath) (interface{}, interface{}, bool) { + // If the source is nil, exit early. + if source == nil { + return nil, nil, false + } + + // Lookup the value by each path segment. + var parent interface{} + v := source + for _, key := range p { + parent = v + switch t := v.(type) { + case []interface{}: + index, ok := key.(int) + if !ok || index < 0 || index >= len(t) { + return nil, nil, false + } + v = t[index] + case map[string]interface{}: + k, ok := key.(string) + if !ok { + return nil, nil, false + } + v, ok = t[k] + if !ok { + return nil, nil, false + } + default: + return nil, nil, false + } + } + return parent, v, true +} + +// getValue returns the value in the container for the given key. +func getValue(container, key interface{}) (interface{}, error) { + switch t := container.(type) { + case []interface{}: + i, ok := key.(int) + contract.Assertf(ok, "key for an array must be an int") + // We explicitly allow i == len(t) here, which indicates a + // value that will be appended to the end of the array. + if i < 0 || i > len(t) { + return nil, errors.New("array index out of range") + } + if i == len(t) { + return nil, nil + } + return t[i], nil + case map[string]interface{}: + k, ok := key.(string) + contract.Assertf(ok, "key for a map must be a string") + return t[k], nil + } + + contract.Failf("should not reach here") + return nil, nil +} + +// Set value sets the value in the container for the given key, and returns the container. +// If the container is an array, and a value is being appended, containerParent and containerParentKey must +// not be nil since a new slice will be created to append the value, which needs to be saved in the parent. +// In this case, the new slice will be returned. +func setValue(container, key, value, containerParent, containerParentKey interface{}) (interface{}, error) { + switch t := container.(type) { + case []interface{}: + i, ok := key.(int) + contract.Assertf(ok, "key for an array must be an int") + // We allow i == len(t), which indicates the value should be appended to the end of the array. + if i < 0 || i > len(t) { + return nil, errors.New("array index out of range") + } + // If i == len(t), we need to append to the end of the array, which involves creating a new slice + // and saving it in the parent container. + if i == len(t) { + t = append(t, value) + contract.Assertf(containerParent != nil, "parent must not be nil") + contract.Assertf(containerParentKey != nil, "parentKey must not be nil") + if _, err := setValue(containerParent, containerParentKey, t, nil, nil); err != nil { + return nil, err + } + return t, nil + } + t[i] = value + case map[string]interface{}: + k, ok := key.(string) + contract.Assertf(ok, "key for a map must be a string") + t[k] = value + } + return container, nil +} + +// adjustObjectValue returns a more suitable value for objects: +func adjustObjectValue(v Value, path bool) interface{} { + contract.Assertf(!v.Object(), "v must not be an Object") + + // If the path flag isn't set, just return the value as-is. + if !path { + return v + } + + // If it's a secure value, return as-is. + if v.Secure() { + return v + } + + // If "false" or "true", return the boolean value. + if v.value == "false" { + return false + } else if v.value == "true" { + return true + } + + // If it's convertible to an int, return the int. + i, err := strconv.Atoi(v.value) + if err == nil { + return i + } + + // Otherwise, just return the string value. + return v.value +} diff --git a/pkg/resource/config/map_test.go b/pkg/resource/config/map_test.go index 2f2fa94e2..818cacd2f 100644 --- a/pkg/resource/config/map_test.go +++ b/pkg/resource/config/map_test.go @@ -16,6 +16,7 @@ package config import ( "encoding/json" + "fmt" "testing" "github.com/pulumi/pulumi/pkg/util/contract" @@ -25,8 +26,8 @@ import ( func TestMarshalMapJSON(t *testing.T) { m := Map{ - Key{namespace: "my", name: "testKey"}: NewValue("testValue"), - Key{namespace: "my", name: "anotherTestKey"}: NewValue("anotherTestValue"), + MustMakeKey("my", "testKey"): NewValue("testValue"), + MustMakeKey("my", "anotherTestKey"): NewValue("anotherTestValue"), } b, err := json.Marshal(m) @@ -43,8 +44,8 @@ func TestMarshalMapJSON(t *testing.T) { func TestMarshalMapYAML(t *testing.T) { m := Map{ - Key{namespace: "my", name: "testKey"}: NewValue("testValue"), - Key{namespace: "my", name: "anotherTestKey"}: NewValue("anotherTestValue"), + MustMakeKey("my", "testKey"): NewValue("testValue"), + MustMakeKey("my", "anotherTestKey"): NewValue("anotherTestValue"), } b, err := yaml.Marshal(m) @@ -59,6 +60,1020 @@ func TestMarshalMapYAML(t *testing.T) { assert.Equal(t, m, newM) } +func TestMarshalling(t *testing.T) { + tests := []struct { + Value map[string]interface{} + Expected Map + }{ + { + Value: map[string]interface{}{ + "my:anotherTestKey": "anotherTestValue", + "my:testKey": "testValue", + }, + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + MustMakeKey("my", "anotherTestKey"): NewValue("anotherTestValue"), + }, + }, + { + Value: map[string]interface{}{ + "my:secureTestKey": map[string]interface{}{ + "secure": "securevalue", + }, + }, + Expected: Map{ + MustMakeKey("my", "secureTestKey"): NewSecureValue("securevalue"), + }, + }, + { + Value: map[string]interface{}{ + "my:arrayKey": []string{"a", "b", "c"}, + }, + Expected: Map{ + MustMakeKey("my", "arrayKey"): NewObjectValue(`["a","b","c"]`), + }, + }, + { + Value: map[string]interface{}{ + "my:mapKey": map[string]interface{}{ + "a": "b", + "c": "d", + }, + }, + Expected: Map{ + MustMakeKey("my", "mapKey"): NewObjectValue(`{"a":"b","c":"d"}`), + }, + }, + { + Value: map[string]interface{}{ + "my:servers": []interface{}{ + map[string]interface{}{"port": 80, "host": "example"}, + }, + }, + Expected: Map{ + MustMakeKey("my", "servers"): NewObjectValue(`[{"host":"example","port":80}]`), + }, + }, + { + Value: map[string]interface{}{ + "my:mapKey": map[string][]int{ + "nums": {1, 2, 3}, + }, + }, + Expected: Map{ + MustMakeKey("my", "mapKey"): NewObjectValue(`{"nums":[1,2,3]}`), + }, + }, + { + Value: map[string]interface{}{ + "my:mapKey": map[string]interface{}{ + "a": map[string]interface{}{"secure": "securevalue"}, + "c": "d", + }, + }, + Expected: Map{ + MustMakeKey("my", "mapKey"): NewSecureObjectValue(`{"a":{"secure":"securevalue"},"c":"d"}`), + }, + }, + { + Value: map[string]interface{}{ + "my:servers": []interface{}{ + map[string]interface{}{ + "port": 80, + "host": "example", + "token": map[string]interface{}{ + "secure": "securevalue", + }, + }, + }, + }, + Expected: Map{ + Key{ + namespace: "my", + name: "servers", + }: NewSecureObjectValue(`[{"host":"example","port":80,"token":{"secure":"securevalue"}}]`), + }, + }, + { + Value: map[string]interface{}{ + "my:mapKey": map[string]interface{}{ + "a": map[string]interface{}{"secure": "foo", "bar": "blah"}, + "c": "d", + }, + }, + Expected: Map{ + MustMakeKey("my", "mapKey"): NewObjectValue(`{"a":{"bar":"blah","secure":"foo"},"c":"d"}`), + }, + }, + } + + for _, test := range tests { + yamlBytes, err := yaml.Marshal(test.Value) + assert.NoError(t, err) + t.Run(fmt.Sprintf("YAML: %s", yamlBytes), func(t *testing.T) { + var m Map + err := yaml.Unmarshal(yamlBytes, &m) + + assert.NoError(t, err) + assert.Equal(t, test.Expected, m) + + newM, err := roundtripMapYAML(m) + assert.NoError(t, err) + assert.Equal(t, m, newM) + }) + + jsonBytes, err := json.Marshal(test.Value) + assert.NoError(t, err) + t.Run(fmt.Sprintf("JSON: %s", jsonBytes), func(t *testing.T) { + var m Map + err := json.Unmarshal(jsonBytes, &m) + + assert.NoError(t, err) + assert.Equal(t, test.Expected, m) + + newM, err := roundtripMapJSON(m) + assert.NoError(t, err) + assert.Equal(t, m, newM) + }) + } +} + +func TestDecrypt(t *testing.T) { + tests := []struct { + Config Map + Expected map[Key]string + }{ + { + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + Expected: map[Key]string{ + MustMakeKey("my", "testKey"): "testValue", + }, + }, + { + Config: Map{ + MustMakeKey("my", "testKey"): NewSecureValue("securevalue"), + }, + Expected: map[Key]string{ + MustMakeKey("my", "testKey"): "[secret]", + }, + }, + { + Config: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`{"inner":"value"}`), + }, + Expected: map[Key]string{ + MustMakeKey("my", "testKey"): `{"inner":"value"}`, + }, + }, + { + Config: Map{ + MustMakeKey("my", "testKey"): NewSecureObjectValue(`{"inner":{"secure":"securevalue"}}`), + }, + Expected: map[Key]string{ + MustMakeKey("my", "testKey"): `{"inner":"[secret]"}`, + }, + }, + { + Config: Map{ + //nolint:lll + MustMakeKey("my", "testKey"): NewSecureObjectValue(`[{"inner":{"secure":"securevalue"}},{"secure":"securevalue2"}]`), + }, + Expected: map[Key]string{ + MustMakeKey("my", "testKey"): `[{"inner":"[secret]"},"[secret]"]`, + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + decrypter := NewBlindingDecrypter() + actual, err := test.Config.Decrypt(decrypter) + assert.NoError(t, err) + assert.Equal(t, test.Expected, actual) + }) + } +} + +func TestGetSuccess(t *testing.T) { + tests := []struct { + Key string + Path bool + Config Map + Expected Value + ExpectNotFound bool + }{ + { + Key: "my:testKey", + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + Expected: NewValue("testValue"), + }, + { + Key: "my:testKey", + Config: Map{ + MustMakeKey("my", "testKey"): NewSecureValue("secureValue"), + }, + Expected: NewSecureValue("secureValue"), + }, + { + Key: "my:test.Key", + Config: Map{ + MustMakeKey("my", "test.Key"): NewValue("testValue"), + }, + Expected: NewValue("testValue"), + }, + { + Key: "my:0", + Path: true, + Config: Map{ + MustMakeKey("my", "0"): NewValue("testValue"), + }, + Expected: NewValue("testValue"), + }, + { + Key: `my:["testKey"]`, + Path: true, + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + Expected: NewValue("testValue"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + Expected: NewValue("value"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":{"secure":"securevalue"}}`), + }, + Expected: NewSecureValue("securevalue"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":true}`), + }, + Expected: NewValue("true"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":false}`), + }, + Expected: NewValue("false"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":100}`), + }, + Expected: NewValue("100"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":-2}`), + }, + Expected: NewValue("-2"), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":{"nested":"foo"}}`), + }, + Expected: NewObjectValue(`{"nested":"foo"}`), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":{"nested":{"secure":"securevalue"}}}`), + }, + Expected: NewSecureObjectValue(`{"nested":{"secure":"securevalue"}}`), + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":{"nested":{"a":"b","secure":"val"}}}`), + }, + Expected: NewObjectValue(`{"nested":{"a":"b","secure":"val"}}`), + }, + { + Key: `my:testKey`, + Path: true, + Config: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`["a"]`), + }, + Expected: NewObjectValue(`["a"]`), + }, + { + Key: `my:names[0]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: NewValue("a"), + }, + { + Key: `my:names[1]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: NewValue("b"), + }, + { + Key: `my:names[2]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: NewValue("c"), + }, + { + Key: `my:names[3]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + ExpectNotFound: true, + }, + { + Key: `my:outer.inner.nested`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"hi"}`), + }, + ExpectNotFound: true, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + key, err := ParseKey(test.Key) + assert.NoError(t, err) + + v, ok, err := test.Config.Get(key, test.Path) + assert.NoError(t, err) + if test.ExpectNotFound { + assert.False(t, ok) + assert.Equal(t, Value{}, v) + } else { + assert.True(t, ok) + assert.Equal(t, test.Expected, v) + } + }) + } +} + +func TestGetFail(t *testing.T) { + tests := []struct { + Key string + }{ + { + Key: `my:["foo`, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test.Key), func(t *testing.T) { + config := make(Map) + + key, err := ParseKey(test.Key) + assert.NoError(t, err) + + _, found, err := config.Get(key, true /*path*/) + assert.False(t, found) + assert.Error(t, err) + }) + } +} + +func TestRemoveSuccess(t *testing.T) { + tests := []struct { + Key string + Path bool + Config Map + Expected Map + }{ + { + Key: "my:testKey", + Config: Map{}, + Expected: Map{}, + }, + { + Key: "my:testKey", + Config: Map{ + MustMakeKey("my", "anotherTestKey"): NewValue("testValue"), + }, + Expected: Map{ + MustMakeKey("my", "anotherTestKey"): NewValue("testValue"), + }, + }, + { + Key: "my:testKey", + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + Expected: Map{}, + }, + { + Key: "my:anotherTestKey", + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + MustMakeKey("my", "anotherTestKey"): NewValue("anotherTestValue"), + }, + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + }, + { + Key: "my:testKey", + Config: Map{ + MustMakeKey("my", "testKey"): NewSecureValue("secureValue"), + }, + Expected: Map{}, + }, + { + Key: `my:outer`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + Expected: Map{}, + }, + { + Key: `my:outer.inner`, + Path: true, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{}`), + }, + }, + { + Key: `my:names[0]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["b","c"]`), + }, + }, + { + Key: `my:names[1]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","c"]`), + }, + }, + { + Key: `my:names[2]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b"]`), + }, + }, + { + Key: `my:names[3]`, + Path: true, + Config: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "names"): NewObjectValue(`["a","b","c"]`), + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + key, err := ParseKey(test.Key) + assert.NoError(t, err) + err = test.Config.Remove(key, test.Path) + assert.NoError(t, err) + assert.Equal(t, test.Expected, test.Config) + }) + } +} + +func TestRemoveFail(t *testing.T) { + tests := []struct { + Key string + Config Map + }{ + { + Key: `my:["foo`, + Config: Map{}, + }, + { + Key: `my:foo.bar`, + Config: Map{ + MustMakeKey("my", "foo"): NewObjectValue(`{"bar":"baz","secure":"myvalue"}`), + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + key, err := ParseKey(test.Key) + assert.NoError(t, err) + + err = test.Config.Remove(key, true /*path*/) + assert.Error(t, err) + }) + } +} + +func TestSetSuccess(t *testing.T) { + tests := []struct { + Key string + Value Value + Path bool + Config Map + Expected Map + }{ + { + Key: "my:testKey", + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + }, + { + Key: "my:anotherTestKey", + Value: NewValue("anotherTestValue"), + Config: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + MustMakeKey("my", "anotherTestKey"): NewValue("anotherTestValue"), + }, + }, + { + Key: "my:0", + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "0"): NewValue("testValue"), + }, + }, + { + Key: "my:true", + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "true"): NewValue("testValue"), + }, + }, + { + Key: "my:test.Key", + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "test.Key"): NewValue("testValue"), + }, + }, + { + Key: "my:testKey", + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("testValue"), + }, + }, + { + Key: "my:0", + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "0"): NewValue("testValue"), + }, + }, + { + Key: "my:true", + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "true"): NewValue("testValue"), + }, + }, + { + Key: `my:["0"]`, + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "0"): NewValue("testValue"), + }, + }, + { + Key: `my:["true"]`, + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "true"): NewValue("testValue"), + }, + }, + { + Key: `my:["test.Key"]`, + Path: true, + Value: NewValue("testValue"), + Expected: Map{ + MustMakeKey("my", "test.Key"): NewValue("testValue"), + }, + }, + { + Key: `my:nested["test.Key"]`, + Path: true, + Value: NewValue("value"), + Expected: Map{ + MustMakeKey("my", "nested"): NewObjectValue(`{"test.Key":"value"}`), + }, + }, + { + Key: `my:outer.inner`, + Path: true, + Value: NewValue("value"), + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Key: `my:outer.inner`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "outer"): NewValue("value"), + }, + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Key: `my:outer.inner`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "outer"): NewSecureValue("securevalue"), + }, + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Key: `my:array[0]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "array"): NewValue("value"), + }, + Expected: Map{ + MustMakeKey("my", "array"): NewObjectValue(`["value"]`), + }, + }, + { + Key: `my:array[0]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "array"): NewSecureValue("value"), + }, + Expected: Map{ + MustMakeKey("my", "array"): NewObjectValue(`["value"]`), + }, + }, + { + Key: `my:outer.inner`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"existing":"existingValue"}`), + }, + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"existing":"existingValue","inner":"value"}`), + }, + }, + { + Key: `my:outer.inner`, + Path: true, + Value: NewSecureValue("securevalue"), + Expected: Map{ + MustMakeKey("my", "outer"): NewSecureObjectValue(`{"inner":{"secure":"securevalue"}}`), + }, + }, + { + Key: `my:outer.inner.nested`, + Path: true, + Value: NewValue("value"), + Expected: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":{"nested":"value"}}`), + }, + }, + { + Key: `my:name[0]`, + Path: true, + Value: NewValue("value"), + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["value"]`), + }, + }, + { + Key: `my:name[0][0]`, + Path: true, + Value: NewValue("value"), + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`[["value"]]`), + }, + }, + { + Key: `my:name[0]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["value","b","c"]`), + }, + }, + { + Key: `my:name[1]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","value","c"]`), + }, + }, + { + Key: `my:name[2]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","value"]`), + }, + }, + { + Key: `my:name[3]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c","value"]`), + }, + }, + { + Key: `my:name[3][0]`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c",["value"]]`), + }, + }, + { + Key: `my:name[3][0]nested`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c",[{"nested":"value"}]]`), + }, + }, + { + Key: `my:name[3].foo.bar`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c",{"foo":{"bar":"value"}}]`), + }, + }, + { + Key: `my:servers[0].name`, + Path: true, + Value: NewValue("foo"), + Expected: Map{ + MustMakeKey("my", "servers"): NewObjectValue(`[{"name":"foo"}]`), + }, + }, + { + Key: `my:servers[0].host`, + Path: true, + Value: NewValue("example"), + Config: Map{ + MustMakeKey("my", "servers"): NewObjectValue(`[{"name":"foo"}]`), + }, + Expected: Map{ + MustMakeKey("my", "servers"): NewObjectValue(`[{"host":"example","name":"foo"}]`), + }, + }, + { + Key: `my:name[0]`, + Path: true, + Value: NewSecureValue("securevalue"), + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + Expected: Map{ + MustMakeKey("my", "name"): NewSecureObjectValue(`[{"secure":"securevalue"},"b","c"]`), + }, + }, + { + Key: `my:testKey`, + Value: NewValue("false"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("false"), + }, + }, + { + Key: `my:testKey`, + Value: NewValue("true"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("true"), + }, + }, + { + Key: `my:testKey`, + Value: NewValue("10"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("10"), + }, + }, + { + Key: `my:testKey`, + Value: NewValue("-1"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewValue("-1"), + }, + }, + { + Key: `my:testKey[0]`, + Path: true, + Value: NewValue("false"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`[false]`), + }, + }, + { + Key: `my:testKey[0]`, + Path: true, + Value: NewValue("true"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`[true]`), + }, + }, + { + Key: `my:testKey[0]`, + Path: true, + Value: NewValue("10"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`[10]`), + }, + }, + { + Key: `my:testKey[0]`, + Path: true, + Value: NewValue("-1"), + Expected: Map{ + MustMakeKey("my", "testKey"): NewObjectValue(`[-1]`), + }, + }, + { + Key: `my:key.secure`, + Path: true, + Value: NewValue("value"), + Config: Map{ + MustMakeKey("my", "key"): NewObjectValue(`{"bar":"baz"}`), + }, + Expected: Map{ + MustMakeKey("my", "key"): NewObjectValue(`{"bar":"baz","secure":"value"}`), + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + if test.Config == nil { + test.Config = make(Map) + } + + key, err := ParseKey(test.Key) + assert.NoError(t, err) + + err = test.Config.Set(key, test.Value, test.Path) + assert.NoError(t, err) + + assert.Equal(t, test.Expected, test.Config) + }) + } +} + +func TestSetFail(t *testing.T) { + tests := []struct { + Key string + Config Map + }{ + // Syntax errors. + {Key: "my:root["}, + {Key: `my:root["nested]`}, + {Key: "my:root.array[abc]"}, + {Key: "my:root.[1]"}, + + // First path component must be a string. + {Key: `my:[""]`}, + {Key: "my:[0]"}, + + // Index out of range. + {Key: `my:name[-1]`}, + {Key: `my:name[1]`}, + { + Key: `my:name[4]`, + Config: Map{ + MustMakeKey("my", "name"): NewObjectValue(`["a","b","c"]`), + }, + }, + + // A "secure" key that is a map with a single string value is reserved by the system. + {Key: `my:key.secure`}, + {Key: `my:super.nested.map.secure`}, + + // Type mismatches. + { + Key: `my:outer.inner`, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue("[1,2,3]"), + }, + }, + { + Key: `my:array[0]`, + Config: Map{ + MustMakeKey("my", "array"): NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Key: `my:outer.inner.nested`, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + }, + { + Key: `my:outer.inner[0]`, + Config: Map{ + MustMakeKey("my", "outer"): NewObjectValue(`{"inner":"value"}`), + }, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test), func(t *testing.T) { + if test.Config == nil { + test.Config = make(Map) + } + + key, err := ParseKey(test.Key) + assert.NoError(t, err) + + err = test.Config.Set(key, NewValue("value"), true /*path*/) + assert.Error(t, err) + }) + } +} + func roundtripMapYAML(m Map) (Map, error) { return roundtripMap(m, yaml.Marshal, yaml.Unmarshal) } diff --git a/pkg/resource/config/value.go b/pkg/resource/config/value.go index dcab40be0..d27b2b896 100644 --- a/pkg/resource/config/value.go +++ b/pkg/resource/config/value.go @@ -16,13 +16,16 @@ package config import ( "encoding/json" - "errors" + "fmt" + + "github.com/pkg/errors" ) // Value is a single config value. type Value struct { value string secure bool + object bool } func NewSecureValue(v string) Value { @@ -33,6 +36,14 @@ func NewValue(v string) Value { return Value{value: v, secure: false} } +func NewSecureObjectValue(v string) Value { + return Value{value: v, secure: true, object: true} +} + +func NewObjectValue(v string) Value { + return Value{value: v, secure: false, object: true} +} + // Value fetches the value of this configuration entry, using decrypter to decrypt if necessary. If the value // is a secret and decrypter is nil, or if decryption fails for any reason, a non-nil error is returned. func (c Value) Value(decrypter Decrypter) (string, error) { @@ -42,47 +53,126 @@ func (c Value) Value(decrypter Decrypter) (string, error) { if decrypter == nil { return "", errors.New("non-nil decrypter required for secret") } + if c.object && decrypter != NopDecrypter { + var obj interface{} + if err := json.Unmarshal([]byte(c.value), &obj); err != nil { + return "", err + } + decryptedObj, err := decryptObject(obj, decrypter) + if err != nil { + return "", err + } + json, err := json.Marshal(decryptedObj) + if err != nil { + return "", err + } + return string(json), nil + } return decrypter.DecryptValue(c.value) } +func (c Value) SecureValues(decrypter Decrypter) ([]string, error) { + d := NewTrackingDecrypter(decrypter) + if _, err := c.Value(d); err != nil { + return nil, err + } + return d.SecureValues(), nil +} + func (c Value) Secure() bool { return c.secure } -func (c Value) MarshalJSON() ([]byte, error) { - if !c.secure { - return json.Marshal(c.value) +func (c Value) Object() bool { + return c.object +} + +// ToObject returns the string value (if not an object), or the unmarshalled JSON object (if an object). +func (c Value) ToObject() (interface{}, error) { + if !c.object { + return c.value, nil } - m := make(map[string]string) - m["secure"] = c.value + var v interface{} + err := json.Unmarshal([]byte(c.value), &v) + if err != nil { + return nil, err + } - return json.Marshal(m) + return v, nil +} + +func (c Value) MarshalJSON() ([]byte, error) { + v, err := c.marshalValue() + if err != nil { + return nil, err + } + return json.Marshal(v) } func (c *Value) UnmarshalJSON(b []byte) error { - var m map[string]string - err := json.Unmarshal(b, &m) - if err == nil { - if len(m) != 1 { - return errors.New("malformed secure data") - } - - val, has := m["secure"] - if !has { - return errors.New("malformed secure data") - } - - c.value = val - c.secure = true - return nil - } - - return json.Unmarshal(b, &c.value) + return c.unmarshalValue( + func(v interface{}) error { + return json.Unmarshal(b, v) + }, + func(v interface{}) interface{} { + return v + }) } func (c Value) MarshalYAML() (interface{}, error) { + return c.marshalValue() +} + +func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error { + return c.unmarshalValue(func(v interface{}) error { + return unmarshal(v) + }, interfaceMapToStringMap) +} + +func (c *Value) unmarshalValue(unmarshal func(interface{}) error, fix func(interface{}) interface{}) error { + // First, try to unmarshal as a string. + err := unmarshal(&c.value) + if err == nil { + c.secure = false + c.object = false + return nil + } + + // Otherwise, try to unmarshal as an object. + var obj interface{} + if err = unmarshal(&obj); err != nil { + return errors.Wrapf(err, "malformed config value") + } + + // Fix-up the object (e.g. convert `map[interface{}]interface{}` to `map[string]interface{}`). + obj = fix(obj) + + if is, val := isSecureValue(obj); is { + c.value = val + c.secure = true + c.object = false + return nil + } + + json, err := json.Marshal(obj) + if err != nil { + return errors.Wrapf(err, "marshalling obj") + } + c.value = string(json) + c.secure = hasSecureValue(obj) + c.object = true + return nil +} + +func (c Value) marshalValue() (interface{}, error) { + if c.object { + var obj interface{} + err := json.Unmarshal([]byte(c.value), &obj) + return obj, err + } + if !c.secure { return c.value, nil } @@ -93,24 +183,93 @@ func (c Value) MarshalYAML() (interface{}, error) { return m, nil } -func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error { - var m map[string]string - err := unmarshal(&m) - if err == nil { - if len(m) != 1 { - return errors.New("malformed secure data") +// The unserialized value from YAML needs to be serializable as JSON, but YAML will unmarshal maps as +// `map[interface{}]interface{}` (because it supports bools as keys), which isn't supported by the JSON +// marshaller. To address, when unserializing YAML, we convert `map[interface{}]interface{}` to +// `map[string]interface{}`. +func interfaceMapToStringMap(v interface{}) interface{} { + switch t := v.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for key, val := range t { + m[fmt.Sprintf("%v", key)] = interfaceMapToStringMap(val) } - - val, has := m["secure"] - if !has { - return errors.New("malformed secure data") + return m + case []interface{}: + a := make([]interface{}, len(t)) + for i, val := range t { + a[i] = interfaceMapToStringMap(val) } + return a + } + return v +} - c.value = val - c.secure = true - return nil +// hasSecureValue returns true if the object contains a value that's a `map[string]string` of +// length one with a "secure" key. +func hasSecureValue(v interface{}) bool { + switch t := v.(type) { + case map[string]interface{}: + if is, _ := isSecureValue(t); is { + return true + } + for _, val := range t { + if hasSecureValue(val) { + return true + } + } + case []interface{}: + for _, val := range t { + if hasSecureValue(val) { + return true + } + } + } + return false +} + +// isSecureValue returns true if the object is a `map[string]string` of length one with a "secure" key. +func isSecureValue(v interface{}) (bool, string) { + if m, isMap := v.(map[string]interface{}); isMap && len(m) == 1 { + if val, hasSecureKey := m["secure"]; hasSecureKey { + if valString, isString := val.(string); isString { + return true, valString + } + } + } + return false, "" +} + +// decryptObject returns a new object with all secure values in the object converted to decrypted strings. +func decryptObject(v interface{}, decrypter Decrypter) (interface{}, error) { + decryptIt := func(val interface{}) (interface{}, error) { + if isSecure, secureVal := isSecureValue(val); isSecure { + return decrypter.DecryptValue(secureVal) + } + return decryptObject(val, decrypter) } - c.secure = false - return unmarshal(&c.value) + switch t := v.(type) { + case map[string]interface{}: + m := make(map[string]interface{}) + for key, val := range t { + decrypted, err := decryptIt(val) + if err != nil { + return nil, err + } + m[key] = decrypted + } + return m, nil + case []interface{}: + a := make([]interface{}, len(t)) + for i, val := range t { + decrypted, err := decryptIt(val) + if err != nil { + return nil, err + } + a[i] = decrypted + } + return a, nil + } + return v, nil } diff --git a/pkg/resource/config/value_test.go b/pkg/resource/config/value_test.go index 4adfdf45d..a1bb81181 100644 --- a/pkg/resource/config/value_test.go +++ b/pkg/resource/config/value_test.go @@ -16,6 +16,7 @@ package config import ( "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -70,6 +71,180 @@ func TestMarshallSecureValueJSON(t *testing.T) { assert.Equal(t, v, newV) } +func TestHasSecureValue(t *testing.T) { + tests := []struct { + Value interface{} + Expected bool + }{ + { + Value: []interface{}{"a", "b", "c"}, + Expected: false, + }, + { + Value: map[string]interface{}{ + "foo": "bar", + "hi": map[string]interface{}{"secure": "securevalue", "but": "not"}, + }, + Expected: false, + }, + { + Value: map[string]interface{}{ + "foo": "bar", + "hi": map[string]interface{}{"secure": 1}, + }, + Expected: false, + }, + { + Value: []interface{}{"a", "b", map[string]interface{}{"secure": "securevalue"}}, + Expected: true, + }, + { + Value: map[string]interface{}{ + "foo": "bar", + "hi": map[string]interface{}{"secure": "securevalue"}, + }, + Expected: true, + }, + { + Value: map[string]interface{}{ + "foo": "bar", + "array": []interface{}{"a", "b", map[string]interface{}{"secure": "securevalue"}}, + }, + Expected: true, + }, + { + Value: map[string]interface{}{ + "foo": "bar", + "map": map[string]interface{}{ + "nest": "blah", + "hi": map[string]interface{}{"secure": "securevalue"}, + }, + }, + Expected: true, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) { + jsonBytes, err := json.Marshal(test.Value) + assert.NoError(t, err) + + var val interface{} + err = json.Unmarshal(jsonBytes, &val) + assert.NoError(t, err) + + assert.Equal(t, test.Expected, hasSecureValue(val)) + }) + } +} + +func TestDecryptingValue(t *testing.T) { + tests := []struct { + Value Value + Expected string + }{ + { + Value: NewValue("value"), + Expected: "value", + }, + { + Value: NewValue(`{"foo":"bar"}`), + Expected: `{"foo":"bar"}`, + }, + { + Value: NewValue(`["a","b"]`), + Expected: `["a","b"]`, + }, + { + Value: NewObjectValue(`{"foo":"bar"}`), + Expected: `{"foo":"bar"}`, + }, + { + Value: NewObjectValue(`["a","b"]`), + Expected: `["a","b"]`, + }, + { + Value: NewSecureValue("securevalue"), + Expected: "[secret]", + }, + { + Value: NewSecureObjectValue(`{"foo":{"secure":"securevalue"}}`), + Expected: `{"foo":"[secret]"}`, + }, + { + Value: NewSecureObjectValue(`["a",{"secure":"securevalue"}]`), + Expected: `["a","[secret]"]`, + }, + } + + decrypter := NewBlindingDecrypter() + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) { + actual, err := test.Value.Value(decrypter) + assert.NoError(t, err) + assert.Equal(t, test.Expected, actual) + + // Ensure the same value is returned when the NopDecrypter is used. + actualNop, err := test.Value.Value(NopDecrypter) + assert.NoError(t, err) + assert.Equal(t, test.Value.value, actualNop) + }) + } +} + +type passThroughDecrypter struct{} + +func (d passThroughDecrypter) DecryptValue(ciphertext string) (string, error) { + return ciphertext, nil +} + +func TestSecureValues(t *testing.T) { + tests := []struct { + Value Value + Expected []string + }{ + { + Value: NewValue("value"), + Expected: nil, + }, + { + Value: NewObjectValue(`{"foo":"bar"}`), + Expected: nil, + }, + { + Value: NewObjectValue(`["a","b"]`), + Expected: nil, + }, + { + Value: NewSecureValue("securevalue"), + Expected: []string{"securevalue"}, + }, + { + Value: NewSecureObjectValue(`{"foo":{"secure":"securevalue"}}`), + Expected: []string{"securevalue"}, + }, + { + Value: NewSecureObjectValue(`["a",{"secure":"securevalue"}]`), + Expected: []string{"securevalue"}, + }, + { + Value: NewSecureObjectValue(`["a",{"secure":"alpha"},{"test":{"secure":"beta"}}]`), + Expected: []string{"alpha", "beta"}, + }, + } + + decrypter := passThroughDecrypter{} + + for _, test := range tests { + t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) { + actual, err := test.Value.SecureValues(decrypter) + assert.NoError(t, err) + assert.Equal(t, test.Expected, actual) + }) + } +} + func roundtripValueYAML(v Value) (Value, error) { return roundtripValue(v, yaml.Marshal, yaml.Unmarshal) } diff --git a/pkg/testing/integration/program.go b/pkg/testing/integration/program.go index 098b11aa8..ec09b82bd 100644 --- a/pkg/testing/integration/program.go +++ b/pkg/testing/integration/program.go @@ -124,6 +124,18 @@ type TestStatsReporter interface { ReportCommand(stats TestCommandStats) } +// ConfigValue is used to provide config values to a test program. +type ConfigValue struct { + // The config key to pass to `pulumi config`. + Key string + // The config value to pass to `pulumi config`. + Value string + // Secret indicates that the `--secret` flag should be specified when calling `pulumi config`. + Secret bool + // Path indicates that the `--path` flag should be specified when calling `pulumi config`. + Path bool +} + // ProgramTestOptions provides options for ProgramTest type ProgramTestOptions struct { // Dir is the program directory to test. @@ -133,10 +145,12 @@ type ProgramTestOptions struct { // Map of package names to versions. The test will use the specified versions of these packages instead of what // is declared in `package.json`. Overrides map[string]string - // Map of config keys and values to set (e.g. {"aws:region": "us-east-2"}) + // Map of config keys and values to set (e.g. {"aws:region": "us-east-2"}). Config map[string]string - // Map of secure config keys and values to set on the stack (e.g. {"aws:region": "us-east-2"}) + // Map of secure config keys and values to set (e.g. {"aws:region": "us-east-2"}). Secrets map[string]string + // List of config keys and values to set in order, including Secret and Path options. + OrderedConfig []ConfigValue // SecretsProvider is the optional custom secrets provider to use instead of the default. SecretsProvider string // EditDirs is an optional list of edits to apply to the example, as subsequent deployments. @@ -923,6 +937,19 @@ func (pt *programTester) testLifeCycleInitialize(dir string) error { } } + for _, cv := range pt.opts.OrderedConfig { + configArgs := []string{"config", "set", cv.Key, cv.Value} + if cv.Secret { + configArgs = append(configArgs, "--secret") + } + if cv.Path { + configArgs = append(configArgs, "--path") + } + if err := pt.runPulumiCommand("pulumi-config", configArgs, dir); err != nil { + return err + } + } + return nil } diff --git a/pkg/workspace/paths.go b/pkg/workspace/paths.go index 1ad02b019..06f232c21 100644 --- a/pkg/workspace/paths.go +++ b/pkg/workspace/paths.go @@ -31,7 +31,7 @@ import ( const ( // BackupDir is the name of the folder where backup stack information is stored. BackupDir = "backups" - // BookkeepingDir is the name of our bookeeping folder, we store state here (like .git for git). + // BookkeepingDir is the name of our bookkeeping folder, we store state here (like .git for git). BookkeepingDir = ".pulumi" // ConfigDir is the name of the folder that holds local configuration information. ConfigDir = "config" diff --git a/tests/integration/config_basic/dotnet/Program.cs b/tests/integration/config_basic/dotnet/Program.cs index cf15c4da3..eede88b1f 100644 --- a/tests/integration/config_basic/dotnet/Program.cs +++ b/tests/integration/config_basic/dotnet/Program.cs @@ -1,6 +1,8 @@ // Copyright 2016-2019, Pulumi Corporation. All rights reserved. using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Pulumi; @@ -8,23 +10,136 @@ class Program { static Task Main(string[] args) { - return Deployment.RunAsync(() => + return Deployment.RunAsync(() => { var config = new Config("config_basic_dotnet"); - // This value is plaintext and doesn't require encryption. - var value = config.Require("aConfigValue"); - if (value != "this value is a value") + var tests = new[] { - throw new Exception($"aConfigValue not the expected value; got {value}"); - } + new Test + { + Key = "aConfigValue", + Expected = "this value is a value" + }, + new Test + { + Key = "bEncryptedSecret", + Expected = "this super secret is encrypted" + }, + new Test + { + Key = "outer", + Expected = "{\"inner\":\"value\"}", + AdditionalValidation = () => + { + var outer = config.RequireObject>("outer"); + if (outer.Count != 1 || outer["inner"] != "value") + { + throw new Exception("'outer' not the expected object value"); + } + } + }, + new Test + { + Key = "names", + Expected = "[\"a\",\"b\",\"c\",\"super secret name\"]", + AdditionalValidation = () => + { + var expected = new[] { "a", "b", "c", "super secret name" }; + var names = config.RequireObject("names"); + if (!Enumerable.SequenceEqual(expected, names)) + { + throw new Exception("'names' not the expected object value"); + } + } + }, + new Test + { + Key = "servers", + Expected = "[{\"host\":\"example\",\"port\":80}]", + AdditionalValidation = () => + { + var servers = config.RequireObject("servers"); + if (servers.Length != 1 || servers[0].host != "example" || servers[0].port != 80) + { + throw new Exception("'servers' not the expected object value"); + } + } + }, + new Test + { + Key = "a", + Expected = "{\"b\":[{\"c\":true},{\"c\":false}]}", + AdditionalValidation = () => + { + var a = config.RequireObject("a"); + if (a.b.Length != 2 || a.b[0].c != true || a.b[1].c != false) + { + throw new Exception("'a' not the expected object value"); + } + } + }, + new Test + { + Key = "tokens", + Expected = "[\"shh\"]", + AdditionalValidation = () => + { + var expected = new[] { "shh" }; + var tokens = config.RequireObject("tokens"); + if (!Enumerable.SequenceEqual(expected, tokens)) + { + throw new Exception("'tokens' not the expected object value"); + } + } + }, + new Test + { + Key = "foo", + Expected = "{\"bar\":\"don't tell\"}", + AdditionalValidation = () => + { + var foo = config.RequireObject>("foo"); + if (foo.Count != 1 || foo["bar"] != "don't tell") + { + throw new Exception("'foo' not the expected object value"); + } + } + }, + }; - // This value is a secret - var secret = config.Require("bEncryptedSecret"); - if (secret != "this super secret is encrypted") + foreach (var test in tests) { - throw new Exception($"bEncryptedSecret not the expected value; got {secret}"); + var value = config.Require(test.Key); + if (value != test.Expected) + { + throw new Exception($"'{test.Key}' not the expected value; got {value}"); + } + test.AdditionalValidation?.Invoke(); } }); } -} \ No newline at end of file +} + +class Test +{ + public string Key; + public string Expected; + public Action AdditionalValidation; +} + +class Server +{ + public string host { get; set; } + public int port { get; set; } +} + +class A +{ + public B[] b { get; set; } +} + +class B +{ + public bool c { get; set; } +} diff --git a/tests/integration/config_basic/go/main.go b/tests/integration/config_basic/go/main.go index c73bcd7e3..798d22a19 100644 --- a/tests/integration/config_basic/go/main.go +++ b/tests/integration/config_basic/go/main.go @@ -4,6 +4,7 @@ package main import ( "fmt" + "github.com/pulumi/pulumi/sdk/go/pulumi" "github.com/pulumi/pulumi/sdk/go/pulumi/config" ) @@ -13,16 +14,49 @@ func main() { // Just test that basic config works. cfg := config.New(ctx, "config_basic_go") - // This value is plaintext and doesn't require encryption. - value := cfg.Require("aConfigValue") - if value != "this value is a value" { - return fmt.Errorf("aConfigValue not the expected value; got %s", value) + tests := []struct { + Key string + Expected string + }{ + { + Key: "aConfigValue", + Expected: `this value is a value`, + }, + { + Key: "bEncryptedSecret", + Expected: `this super secret is encrypted`, + }, + { + Key: "outer", + Expected: `{"inner":"value"}`, + }, + { + Key: "names", + Expected: `["a","b","c","super secret name"]`, + }, + { + Key: "servers", + Expected: `[{"host":"example","port":80}]`, + }, + { + Key: "a", + Expected: `{"b":[{"c":true},{"c":false}]}`, + }, + { + Key: "tokens", + Expected: `["shh"]`, + }, + { + Key: "foo", + Expected: `{"bar":"don't tell"}`, + }, } - // This value is a secret and is encrypted using the passphrase `supersecret`. - secret := cfg.Require("bEncryptedSecret") - if secret != "this super secret is encrypted" { - return fmt.Errorf("bEncryptedSecret not the expected value; got %s", secret) + for _, test := range tests { + value := cfg.Require(test.Key) + if value != test.Expected { + return fmt.Errorf("%q not the expected value; got %q", test.Key, value) + } } return nil diff --git a/tests/integration/config_basic/nodejs/index.ts b/tests/integration/config_basic/nodejs/index.ts index ac1bb88a8..8fb1027d2 100644 --- a/tests/integration/config_basic/nodejs/index.ts +++ b/tests/integration/config_basic/nodejs/index.ts @@ -8,8 +8,52 @@ const config = new Config("config_basic_js"); // This value is plaintext and doesn't require encryption. const value = config.require("aConfigValue"); -assert.equal(value, "this value is a value", "aConfigValue not the expected value"); +assert.strictEqual(value, "this value is a value", "'aConfigValue' not the expected value"); // This value is a secret and is encrypted using the passphrase `supersecret`. const secret = config.require("bEncryptedSecret"); -assert.equal(secret, "this super secret is encrypted", "bEncryptedSecret not the expected value"); +assert.strictEqual(secret, "this super secret is encrypted", "'bEncryptedSecret' not the expected value"); + +const testData: { + key: string; + expectedJSON: string; + expectedObject: any; +}[] = [ + { + key: "outer", + expectedJSON: `{"inner":"value"}`, + expectedObject: { inner: "value" }, + }, + { + key: "names", + expectedJSON: `["a","b","c","super secret name"]`, + expectedObject: ["a", "b", "c", "super secret name"], + }, + { + key: "servers", + expectedJSON: `[{"host":"example","port":80}]`, + expectedObject: [{ host: "example", port: 80 }], + }, + { + key: "a", + expectedJSON: `{"b":[{"c":true},{"c":false}]}`, + expectedObject: { b: [{ c: true }, { c: false }] }, + }, + { + key: "tokens", + expectedJSON: `["shh"]`, + expectedObject: ["shh"], + }, + { + key: "foo", + expectedJSON: `{"bar":"don't tell"}`, + expectedObject: { bar: "don't tell" }, + }, +]; + +for (const test of testData) { + const json = config.require(test.key); + const obj = config.requireObject(test.key); + assert.strictEqual(json, test.expectedJSON, `'${test.key}' not the expected JSON`); + assert.deepStrictEqual(obj, test.expectedObject, `'${test.key}' not the expected object`); +} diff --git a/tests/integration/config_basic/python/__main__.py b/tests/integration/config_basic/python/__main__.py index 2216c2a9c..9f0b25ea2 100644 --- a/tests/integration/config_basic/python/__main__.py +++ b/tests/integration/config_basic/python/__main__.py @@ -12,3 +12,42 @@ assert value == 'this value is a Pythonic value' # This value is a secret and is encrypted using the passphrase `supersecret`. secret = config.require('bEncryptedSecret') assert secret == 'this super Pythonic secret is encrypted' + +test_data = [ + { + 'key': 'outer', + 'expected_json': '{"inner":"value"}', + 'expected_object': { 'inner': 'value' } + }, + { + 'key': 'names', + 'expected_json': '["a","b","c","super secret name"]', + 'expected_object': ['a', 'b', 'c', 'super secret name'] + }, + { + 'key': 'servers', + 'expected_json': '[{"host":"example","port":80}]', + 'expected_object': [{ 'host': 'example', 'port': 80 }] + }, + { + 'key': 'a', + 'expected_json': '{"b":[{"c":true},{"c":false}]}', + 'expected_object': { 'b': [{ 'c': True }, { 'c': False }] } + }, + { + 'key': 'tokens', + 'expected_json': '["shh"]', + 'expected_object': ['shh'] + }, + { + 'key': 'foo', + 'expected_json': '{"bar":"don\'t tell"}', + 'expected_object': { 'bar': "don't tell" } + } +] + +for test in test_data: + json = config.require(test['key']) + obj = config.require_object(test['key']) + assert json == test['expected_json'] + assert obj == test['expected_object'] diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index d03e811c4..ccd3e5195 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -562,6 +562,301 @@ func TestConfigSave(t *testing.T) { e.RunCommand("pulumi", "stack", "rm", "--yes") } +// TestConfigPaths ensures that config commands with paths work as expected. +func TestConfigPaths(t *testing.T) { + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + // Initialize an empty stack. + path := filepath.Join(e.RootPath, "Pulumi.yaml") + err := (&workspace.Project{ + Name: "testing-config", + Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil), + }).Save(path) + assert.NoError(t, err) + e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "testing") + + namespaces := []string{"", "my:"} + + tests := []struct { + Key string + Value string + Secret bool + Path bool + TopLevelKey string + TopLevelExpectedValue string + }{ + { + Key: "aConfigValue", + Value: "this value is a value", + TopLevelKey: "aConfigValue", + TopLevelExpectedValue: "this value is a value", + }, + { + Key: "anotherConfigValue", + Value: "this value is another value", + TopLevelKey: "anotherConfigValue", + TopLevelExpectedValue: "this value is another value", + }, + { + Key: "bEncryptedSecret", + Value: "this super secret is encrypted", + Secret: true, + TopLevelKey: "bEncryptedSecret", + TopLevelExpectedValue: "this super secret is encrypted", + }, + { + Key: "anotherEncryptedSecret", + Value: "another encrypted secret", + Secret: true, + TopLevelKey: "anotherEncryptedSecret", + TopLevelExpectedValue: "another encrypted secret", + }, + { + Key: "[]", + Value: "square brackets value", + TopLevelKey: "[]", + TopLevelExpectedValue: "square brackets value", + }, + { + Key: "x.y", + Value: "x.y value", + TopLevelKey: "x.y", + TopLevelExpectedValue: "x.y value", + }, + { + Key: "0", + Value: "0 value", + Path: true, + TopLevelKey: "0", + TopLevelExpectedValue: "0 value", + }, + { + Key: "true", + Value: "value", + Path: true, + TopLevelKey: "true", + TopLevelExpectedValue: "value", + }, + { + Key: `["test.Key"]`, + Value: "test key value", + Path: true, + TopLevelKey: "test.Key", + TopLevelExpectedValue: "test key value", + }, + { + Key: `nested["test.Key"]`, + Value: "nested test key value", + Path: true, + TopLevelKey: "nested", + TopLevelExpectedValue: `{"test.Key":"nested test key value"}`, + }, + { + Key: "outer.inner", + Value: "value", + Path: true, + TopLevelKey: "outer", + TopLevelExpectedValue: `{"inner":"value"}`, + }, + { + Key: "names[0]", + Value: "a", + Path: true, + TopLevelKey: "names", + TopLevelExpectedValue: `["a"]`, + }, + { + Key: "names[1]", + Value: "b", + Path: true, + TopLevelKey: "names", + TopLevelExpectedValue: `["a","b"]`, + }, + { + Key: "names[2]", + Value: "c", + Path: true, + TopLevelKey: "names", + TopLevelExpectedValue: `["a","b","c"]`, + }, + { + Key: "names[3]", + Value: "super secret name", + Path: true, + Secret: true, + TopLevelKey: "names", + TopLevelExpectedValue: `["a","b","c","super secret name"]`, + }, + { + Key: "servers[0].port", + Value: "80", + Path: true, + TopLevelKey: "servers", + TopLevelExpectedValue: `[{"port":80}]`, + }, + { + Key: "servers[0].host", + Value: "example", + Path: true, + TopLevelKey: "servers", + TopLevelExpectedValue: `[{"host":"example","port":80}]`, + }, + { + Key: "a.b[0].c", + Value: "true", + Path: true, + TopLevelKey: "a", + TopLevelExpectedValue: `{"b":[{"c":true}]}`, + }, + { + Key: "a.b[1].c", + Value: "false", + Path: true, + TopLevelKey: "a", + TopLevelExpectedValue: `{"b":[{"c":true},{"c":false}]}`, + }, + { + Key: "tokens[0]", + Value: "shh", + Path: true, + Secret: true, + TopLevelKey: "tokens", + TopLevelExpectedValue: `["shh"]`, + }, + { + Key: "foo.bar", + Value: "don't tell", + Path: true, + Secret: true, + TopLevelKey: "foo", + TopLevelExpectedValue: `{"bar":"don't tell"}`, + }, + { + Key: "semiInner.a.b.c.d", + Value: "1", + Path: true, + TopLevelKey: "semiInner", + TopLevelExpectedValue: `{"a":{"b":{"c":{"d":1}}}}`, + }, + { + Key: "wayInner.a.b.c.d.e.f.g.h.i.j.k", + Value: "false", + Path: true, + TopLevelKey: "wayInner", + TopLevelExpectedValue: `{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{"k":false}}}}}}}}}}}`, + }, + + // Overwriting a top-level string value is allowed. + { + Key: "aConfigValue.inner", + Value: "new value", + Path: true, + TopLevelKey: "aConfigValue", + TopLevelExpectedValue: `{"inner":"new value"}`, + }, + { + Key: "anotherConfigValue[0]", + Value: "new value", + Path: true, + TopLevelKey: "anotherConfigValue", + TopLevelExpectedValue: `["new value"]`, + }, + { + Key: "bEncryptedSecret.inner", + Value: "new value", + Path: true, + TopLevelKey: "bEncryptedSecret", + TopLevelExpectedValue: `{"inner":"new value"}`, + }, + { + Key: "anotherEncryptedSecret[0]", + Value: "new value", + Path: true, + TopLevelKey: "anotherEncryptedSecret", + TopLevelExpectedValue: `["new value"]`, + }, + } + + validateConfigGet := func(key string, value string, path bool) { + args := []string{"config", "get", key} + if path { + args = append(args, "--path") + } + stdout, stderr := e.RunCommand("pulumi", args...) + assert.Equal(t, fmt.Sprintf("%s\n", value), stdout) + assert.Equal(t, "", stderr) + } + + for _, ns := range namespaces { + for _, test := range tests { + key := fmt.Sprintf("%s%s", ns, test.Key) + topLevelKey := fmt.Sprintf("%s%s", ns, test.TopLevelKey) + + // Set the value. + args := []string{"config", "set"} + if test.Secret { + args = append(args, "--secret") + } + if test.Path { + args = append(args, "--path") + } + args = append(args, key, test.Value) + stdout, stderr := e.RunCommand("pulumi", args...) + assert.Equal(t, "", stdout) + assert.Equal(t, "", stderr) + + // Get the value and validate it. + validateConfigGet(key, test.Value, test.Path) + + // Get the top-level value and validate it. + validateConfigGet(topLevelKey, test.TopLevelExpectedValue, false /*path*/) + } + } + + badKeys := []string{ + // Syntax errors. + "root[", + `root["nested]`, + "root.array[abc]", + "root.[1]", + + // First path segment must be a non-empty string. + `[""]`, + "[0]", + + // Index out of range. + "names[-1]", + "names[5]", + + // A "secure" key that is a map with a single string value is reserved by the system. + "key.secure", + "super.nested.map.secure", + + // Type mismatch. + "outer[0]", + "names.nested", + "outer.inner.nested", + "outer.inner[0]", + } + + for _, ns := range namespaces { + for _, badKey := range badKeys { + key := fmt.Sprintf("%s%s", ns, badKey) + stdout, stderr := e.RunCommandExpectError("pulumi", "config", "set", "--path", key, "value") + assert.Equal(t, "", stdout) + assert.NotEqual(t, "", stderr) + } + } + + e.RunCommand("pulumi", "stack", "rm", "--yes") +} + // Tests basic configuration from the perspective of a Pulumi program. func TestConfigBasicNodeJS(t *testing.T) { integration.ProgramTest(t, &integration.ProgramTestOptions{ @@ -574,6 +869,19 @@ func TestConfigBasicNodeJS(t *testing.T) { Secrets: map[string]string{ "bEncryptedSecret": "this super secret is encrypted", }, + OrderedConfig: []integration.ConfigValue{ + {Key: "outer.inner", Value: "value", Path: true}, + {Key: "names[0]", Value: "a", Path: true}, + {Key: "names[1]", Value: "b", Path: true}, + {Key: "names[2]", Value: "c", Path: true}, + {Key: "names[3]", Value: "super secret name", Path: true, Secret: true}, + {Key: "servers[0].port", Value: "80", Path: true}, + {Key: "servers[0].host", Value: "example", Path: true}, + {Key: "a.b[0].c", Value: "true", Path: true}, + {Key: "a.b[1].c", Value: "false", Path: true}, + {Key: "tokens[0]", Value: "shh", Path: true, Secret: true}, + {Key: "foo.bar", Value: "don't tell", Path: true, Secret: true}, + }, }) } @@ -599,9 +907,11 @@ func TestInvalidVersionInPackageJson(t *testing.T) { // Tests basic configuration from the perspective of a Pulumi program. func TestConfigBasicPython(t *testing.T) { - t.Skip("pulumi/pulumi#2138") integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join("config_basic", "python"), + Dir: filepath.Join("config_basic", "python"), + Dependencies: []string{ + path.Join("..", "..", "sdk", "python", "env", "src"), + }, Quick: true, Config: map[string]string{ "aConfigValue": "this value is a Pythonic value", @@ -609,6 +919,19 @@ func TestConfigBasicPython(t *testing.T) { Secrets: map[string]string{ "bEncryptedSecret": "this super Pythonic secret is encrypted", }, + OrderedConfig: []integration.ConfigValue{ + {Key: "outer.inner", Value: "value", Path: true}, + {Key: "names[0]", Value: "a", Path: true}, + {Key: "names[1]", Value: "b", Path: true}, + {Key: "names[2]", Value: "c", Path: true}, + {Key: "names[3]", Value: "super secret name", Path: true, Secret: true}, + {Key: "servers[0].port", Value: "80", Path: true}, + {Key: "servers[0].host", Value: "example", Path: true}, + {Key: "a.b[0].c", Value: "true", Path: true}, + {Key: "a.b[1].c", Value: "false", Path: true}, + {Key: "tokens[0]", Value: "shh", Path: true, Secret: true}, + {Key: "foo.bar", Value: "don't tell", Path: true, Secret: true}, + }, }) } @@ -623,6 +946,19 @@ func TestConfigBasicGo(t *testing.T) { Secrets: map[string]string{ "bEncryptedSecret": "this super secret is encrypted", }, + OrderedConfig: []integration.ConfigValue{ + {Key: "outer.inner", Value: "value", Path: true}, + {Key: "names[0]", Value: "a", Path: true}, + {Key: "names[1]", Value: "b", Path: true}, + {Key: "names[2]", Value: "c", Path: true}, + {Key: "names[3]", Value: "super secret name", Path: true, Secret: true}, + {Key: "servers[0].port", Value: "80", Path: true}, + {Key: "servers[0].host", Value: "example", Path: true}, + {Key: "a.b[0].c", Value: "true", Path: true}, + {Key: "a.b[1].c", Value: "false", Path: true}, + {Key: "tokens[0]", Value: "shh", Path: true, Secret: true}, + {Key: "foo.bar", Value: "don't tell", Path: true, Secret: true}, + }, }) } @@ -637,6 +973,19 @@ func TestConfigBasicDotNet(t *testing.T) { Secrets: map[string]string{ "bEncryptedSecret": "this super secret is encrypted", }, + OrderedConfig: []integration.ConfigValue{ + {Key: "outer.inner", Value: "value", Path: true}, + {Key: "names[0]", Value: "a", Path: true}, + {Key: "names[1]", Value: "b", Path: true}, + {Key: "names[2]", Value: "c", Path: true}, + {Key: "names[3]", Value: "super secret name", Path: true, Secret: true}, + {Key: "servers[0].port", Value: "80", Path: true}, + {Key: "servers[0].host", Value: "example", Path: true}, + {Key: "a.b[0].c", Value: "true", Path: true}, + {Key: "a.b[1].c", Value: "false", Path: true}, + {Key: "tokens[0]", Value: "shh", Path: true, Secret: true}, + {Key: "foo.bar", Value: "don't tell", Path: true, Secret: true}, + }, }) }