Support lists and maps in config (#3342)

This change adds support for lists and maps in config. We now allow
lists/maps (and nested structures) in `Pulumi.<stack>.yaml` (or
`Pulumi.<stack>.json`; yes, we currently support that).

For example:

```yaml
config:
  proj:blah:
  - a
  - b
  - c
  proj:hello: world
  proj:outer:
    inner: value
  proj:servers:
  - port: 80
```

While such structures could be specified in the `.yaml` file manually,
we support setting values in maps/lists from the command line.

As always, you can specify single values with:

```shell
$ pulumi config set hello world
```

Which results in the following YAML:

```yaml
proj:hello world
```

And single value secrets via:

```shell
$ pulumi config set --secret token shhh
```

Which results in the following YAML:

```yaml
proj:token:
  secure: v1:VZAhuroR69FkEPTk:isKafsoZVMWA9pQayGzbWNynww==
```

Values in a list can be set from the command line using the new
`--path` flag, which indicates the config key contains a path to a
property in a map or list:

```shell
$ pulumi config set --path names[0] a
$ pulumi config set --path names[1] b
$ pulumi config set --path names[2] c
```

Which results in:

```yaml
proj:names
- a
- b
- c
```

Values can be obtained similarly:

```shell
$ pulumi config get --path names[1]
b
```

Or setting values in a map:

```shell
$ pulumi config set --path outer.inner value
```

Which results in:

```yaml
proj:outer:
  inner: value
```

Of course, setting values in nested structures is supported:

```shell
$ pulumi config set --path servers[0].port 80
```

Which results in:

```yaml
proj:servers:
- port: 80
```

If you want to include a period in the name of a property, it can be
specified as:

```
$ pulumi config set --path 'nested["foo.bar"]' baz
```

Which results in:

```yaml
proj:nested:
  foo.bar: baz
```

Examples of valid paths:

- root
- root.nested
- 'root["nested"]'
- root.double.nest
- 'root["double"].nest'
- 'root["double"]["nest"]'
- root.array[0]
- root.array[100]
- root.array[0].nested
- root.array[0][1].nested
- root.nested.array[0].double[1]
- 'root["key with \"escaped\" quotes"]'
- 'root["key with a ."]'
- '["root key with \"escaped\" quotes"].nested'
- '["root key with a ."][100]'

Note: paths that contain quotes can be surrounded by single quotes.

When setting values with `--path`, if the value is `"false"` or
`"true"`, it will be saved as the boolean value, and if it is
convertible to an integer, it will be saved as an integer.

Secure values are supported in lists/maps as well:

```shell
$ pulumi config set --path --secret tokens[0] shh
```

Will result in:

```yaml
proj:tokens:
- secure: v1:wpZRCe36sFg1RxwG:WzPeQrCn4n+m4Ks8ps15MxvFXg==
```

Note: maps of length 1 with a key of “secure” and string value are
reserved for storing secret values. Attempting to create such a value
manually will result in an error:

```shell
$ pulumi config set --path parent.secure foo
error: "secure" key in maps of length 1 are reserved
```

**Accessing config values from the command line with JSON**

```shell
$ pulumi config --json
```

Will output:

```json
{
  "proj:hello": {
    "value": "world",
    "secret": false,
    "object": false
  },
  "proj:names": {
    "value": "[\"a\",\"b\",\"c\"]",
    "secret": false,
    "object": true,
    "objectValue": [
      "a",
      "b",
      "c"
    ]
  },
  "proj:nested": {
    "value": "{\"foo.bar\":\"baz\"}",
    "secret": false,
    "object": true,
    "objectValue": {
      "foo.bar": "baz"
    }
  },
  "proj:outer": {
    "value": "{\"inner\":\"value\"}",
    "secret": false,
    "object": true,
    "objectValue": {
      "inner": "value"
    }
  },
  "proj:servers": {
    "value": "[{\"port\":80}]",
    "secret": false,
    "object": true,
    "objectValue": [
      {
        "port": 80
      }
    ]
  },
  "proj:token": {
    "secret": true,
    "object": false
  },
  "proj:tokens": {
    "secret": true,
    "object": true
  }
}
```

If the value is a map or list, `"object"` will be `true`. `"value"` will
contain the object as serialized JSON and a new `"objectValue"` property
will be available containing the value of the object.

If the object contains any secret values, `"secret"` will be `true`, and
just like with scalar values, the value will not be outputted unless
`--show-secrets` is specified.

**Accessing config values from Pulumi programs**

Map/list values are available to Pulumi programs as serialized JSON, so
the existing
`getObject`/`requireObject`/`getSecretObject`/`requireSecretObject`
functions can be used to retrieve such values, e.g.:

```typescript
import * as pulumi from "@pulumi/pulumi";

interface Server {
    port: number;
}

const config = new pulumi.Config();

const names = config.requireObject<string[]>("names");
for (const n of names) {
    console.log(n);
}

const servers = config.requireObject<Server[]>("servers");
for (const s of servers) {
    console.log(s.port);
}
```
This commit is contained in:
Justin Van Patten 2019-11-01 13:41:27 -07:00 committed by GitHub
parent 6900ff5bc5
commit c08714ffb4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 2884 additions and 100 deletions

View file

@ -25,6 +25,9 @@ CHANGELOG
- `pulumi stack` now renders the stack as a tree view.
[#3430](https://github.com/pulumi/pulumi/pull/3430)
- Support for lists and maps in config.
[#3342](https://github.com/pulumi/pulumi/pull/3342)
## 1.4.0 (2019-10-24)
- `FileAsset` in the Python SDK now accepts anything implementing `os.PathLike` in addition to `str`.

View file

@ -86,11 +86,18 @@ func newConfigCmd() *cobra.Command {
func newConfigGetCmd(stack *string) *cobra.Command {
var jsonOut bool
var path bool
getCmd := &cobra.Command{
Use: "get <key>",
Short: "Get a single configuration value",
Args: cmdutil.SpecificArgs([]string{"key"}),
Long: "Get a single configuration value.\n\n" +
"The `--path` flag can be used to get a value inside a map or list:\n\n" +
" - `pulumi config get --path outer.inner` will get the value of the `inner` key, " +
"if the value of `outer` is a map `inner: value`.\n" +
" - `pulumi config get --path names[0]` will get the value of the first item, " +
"if the value of `names` is a list.",
Args: cmdutil.SpecificArgs([]string{"key"}),
Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error {
opts := display.Options{
Color: cmdutil.GetGlobalColorization(),
@ -106,21 +113,32 @@ func newConfigGetCmd(stack *string) *cobra.Command {
return errors.Wrap(err, "invalid configuration key")
}
return getConfig(s, key, jsonOut)
return getConfig(s, key, path, jsonOut)
}),
}
getCmd.Flags().BoolVarP(
&jsonOut, "json", "j", false,
"Emit output as JSON")
getCmd.PersistentFlags().BoolVar(
&path, "path", false,
"The key contains a path to a property in a map or list to get")
return getCmd
}
func newConfigRmCmd(stack *string) *cobra.Command {
var path bool
rmCmd := &cobra.Command{
Use: "rm <key>",
Short: "Remove configuration value",
Args: cmdutil.SpecificArgs([]string{"key"}),
Long: "Remove configuration value.\n\n" +
"The `--path` flag can be used to remove a value inside a map or list:\n\n" +
" - `pulumi config rm --path outer.inner` will remove the `inner` key, " +
"if the value of `outer` is a map `inner: value`.\n" +
" - `pulumi config rm --path names[0]` will remove the first item, " +
"if the value of `names` is a list.",
Args: cmdutil.SpecificArgs([]string{"key"}),
Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error {
opts := display.Options{
Color: cmdutil.GetGlobalColorization(),
@ -141,13 +159,17 @@ func newConfigRmCmd(stack *string) *cobra.Command {
return err
}
if ps.Config != nil {
delete(ps.Config, key)
err = ps.Config.Remove(key, path)
if err != nil {
return err
}
return saveProjectStack(s, ps)
}),
}
rmCmd.PersistentFlags().BoolVar(
&path, "path", false,
"The key contains a path to a property in a map or list to remove")
return rmCmd
}
@ -226,13 +248,19 @@ func newConfigRefreshCmd(stack *string) *cobra.Command {
func newConfigSetCmd(stack *string) *cobra.Command {
var plaintext bool
var secret bool
var path bool
setCmd := &cobra.Command{
Use: "set <key> [value]",
Short: "Set configuration value",
Long: "Configuration values can be accessed when a stack is being deployed and used to configure behavior. \n" +
"If a value is not present on the command line, pulumi will prompt for the value. Multi-line values\n" +
"may be set by piping a file to standard in.",
"may be set by piping a file to standard in.\n\n" +
"The `--path` flag can be used to set a value inside a map or list:\n\n" +
" - `pulumi config set --path outer.inner value` " +
"will set the value of `outer` to a map `inner: value`.\n" +
" - `pulumi config set --path names[0] a` " +
"will set the value to a list with the first item `a`.",
Args: cmdutil.RangeArgs(1, 2),
Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error {
opts := display.Options{
@ -301,12 +329,18 @@ func newConfigSetCmd(stack *string) *cobra.Command {
return err
}
ps.Config[key] = v
err = ps.Config.Set(key, v, path)
if err != nil {
return err
}
return saveProjectStack(s, ps)
}),
}
setCmd.PersistentFlags().BoolVar(
&path, "path", false,
"The key contains a path to a property in a map or list to set")
setCmd.PersistentFlags().BoolVar(
&plaintext, "plaintext", false,
"Save the value as plaintext (unencrypted)")
@ -376,8 +410,10 @@ func prettyKeyForProject(k config.Key, proj *workspace.Project) string {
// structure in the future, we should not change existing fields.
type configValueJSON struct {
// When the value is encrypted and --show-secrets was not passed, the value will not be set.
Value *string `json:"value,omitempty"`
Secret bool `json:"secret"`
// If the value is an object, ObjectValue will be set.
Value *string `json:"value,omitempty"`
ObjectValue interface{} `json:"objectValue,omitempty"`
Secret bool `json:"secret"`
}
func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error {
@ -419,11 +455,20 @@ func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error {
}
entry.Value = &decrypted
if cfg[key].Object() {
var obj interface{}
if err := json.Unmarshal([]byte(decrypted), &obj); err != nil {
return err
}
entry.ObjectValue = obj
}
// If the value was a secret value and we aren't showing secrets, then the above would have set value
// to "[secret]" which is reasonable when printing for human display, but for our JSON output, we'd rather
// just elide the value.
if cfg[key].Secure() && !showSecrets {
entry.Value = nil
entry.ObjectValue = nil
}
configValues[key.String()] = entry
@ -453,7 +498,7 @@ func listConfig(stack backend.Stack, showSecrets bool, jsonOut bool) error {
return nil
}
func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error {
func getConfig(stack backend.Stack, key config.Key, path, jsonOut bool) error {
ps, err := loadProjectStack(stack)
if err != nil {
return err
@ -461,7 +506,11 @@ func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error {
cfg := ps.Config
if v, ok := cfg[key]; ok {
v, ok, err := cfg.Get(key, path)
if err != nil {
return err
}
if ok {
var d config.Decrypter
if v.Secure() {
var err error
@ -482,6 +531,14 @@ func getConfig(stack backend.Stack, key config.Key, jsonOut bool) error {
Secret: v.Secure(),
}
if v.Object() {
var obj interface{}
if err := json.Unmarshal([]byte(raw), &obj); err != nil {
return err
}
value.ObjectValue = obj
}
out, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err

View file

@ -15,6 +15,7 @@
package cmd
import (
"encoding/json"
"fmt"
"sort"
"strings"
@ -123,6 +124,14 @@ func displayUpdatesJSON(updates []backend.UpdateInfo, decrypter config.Decrypter
value, err := v.Value(decrypter)
contract.AssertNoError(err)
configValue.Value = makeStringRef(value)
if v.Object() {
var obj interface{}
if err := json.Unmarshal([]byte(value), &obj); err != nil {
return err
}
configValue.ObjectValue = obj
}
}
info.Config[k.String()] = configValue

View file

@ -49,6 +49,7 @@ type promptForValueFunc func(yes bool, valueType string, defaultValue string, se
type newArgs struct {
configArray []string
configPath bool
description string
dir string
force bool
@ -253,7 +254,8 @@ func runNew(args newArgs) error {
// Prompt for config values (if needed) and save.
if !args.generateOnly {
if err = handleConfig(s, args.templateNameOrURL, template, args.configArray, args.yes, opts); err != nil {
err = handleConfig(s, args.templateNameOrURL, template, args.configArray, args.yes, args.configPath, opts)
if err != nil {
return err
}
}
@ -384,6 +386,9 @@ func newNewCmd() *cobra.Command {
cmd.PersistentFlags().StringArrayVarP(
&args.configArray, "config", "c", []string{},
"Config to save")
cmd.PersistentFlags().BoolVar(
&args.configPath, "config-path", false,
"Config keys contain a path to a property in a map or list to set")
cmd.PersistentFlags().StringVarP(
&args.description, "description", "d", "",
"The project description; if not specified, a prompt will request it")
@ -724,7 +729,7 @@ func chooseTemplate(templates []workspace.Template, opts display.Options) (works
// These are passed as `-c aws:region=us-east-1 -c foo:bar=blah` and end up
// in configArray as ["aws:region=us-east-1", "foo:bar=blah"].
// This function converts the array into a config.Map.
func parseConfig(configArray []string) (config.Map, error) {
func parseConfig(configArray []string, path bool) (config.Map, error) {
configMap := make(config.Map)
for _, c := range configArray {
kvp := strings.SplitN(c, "=", 2)
@ -739,7 +744,9 @@ func parseConfig(configArray []string) (config.Map, error) {
value = config.NewValue(kvp[1])
}
configMap[key] = value
if err = configMap.Set(key, value, path); err != nil {
return nil, err
}
}
return configMap, nil
}

View file

@ -23,6 +23,7 @@ import (
"github.com/pulumi/pulumi/pkg/backend"
"github.com/pulumi/pulumi/pkg/backend/display"
"github.com/pulumi/pulumi/pkg/resource/config"
"github.com/pulumi/pulumi/pkg/workspace"
"github.com/stretchr/testify/assert"
)
@ -448,6 +449,278 @@ func TestInvalidTemplateName(t *testing.T) {
})
}
func TestParseConfigSuccess(t *testing.T) {
tests := []struct {
Array []string
Path bool
Expected config.Map
}{
{
Array: []string{},
Expected: config.Map{},
},
{
Array: []string{"my:testKey"},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue(""),
},
},
{
Array: []string{"my:testKey="},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue(""),
},
},
{
Array: []string{"my:testKey=testValue"},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("testValue"),
},
},
{
Array: []string{"my:testKey=test=Value"},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("test=Value"),
},
},
{
Array: []string{
"my:testKey=testValue",
"my:testKey=rewritten",
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("rewritten"),
},
},
{
Array: []string{
"my:testKey=testValue",
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("testValue"),
},
},
{
Array: []string{
"my:test.Key=testValue",
},
Expected: config.Map{
config.MustMakeKey("my", "test.Key"): config.NewValue("testValue"),
},
},
{
Array: []string{
"my:testKey=testValue",
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("testValue"),
},
},
{
Array: []string{
"my:0=testValue",
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "0"): config.NewValue("testValue"),
},
},
{
Array: []string{
"my:true=testValue",
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "true"): config.NewValue("testValue"),
},
},
{
Array: []string{
`my:["test.Key"]=testValue`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "test.Key"): config.NewValue("testValue"),
},
},
{
Array: []string{
`my:outer.inner=value`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "outer"): config.NewObjectValue(`{"inner":"value"}`),
},
},
{
Array: []string{
`my:outer.inner.nested=value`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "outer"): config.NewObjectValue(`{"inner":{"nested":"value"}}`),
},
},
{
Array: []string{
`my:name[0]=value`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "name"): config.NewObjectValue(`["value"]`),
},
},
{
Array: []string{
`my:name[0][0]=value`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "name"): config.NewObjectValue(`[["value"]]`),
},
},
{
Array: []string{
`my:servers[0].name=foo`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "servers"): config.NewObjectValue(`[{"name":"foo"}]`),
},
},
{
Array: []string{
`my:testKey=false`,
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("false"),
},
},
{
Array: []string{
`my:testKey=true`,
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("true"),
},
},
{
Array: []string{
`my:testKey=10`,
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("10"),
},
},
{
Array: []string{
`my:testKey=-1`,
},
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewValue("-1"),
},
},
{
Array: []string{
`my:testKey[0]=false`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[false]`),
},
},
{
Array: []string{
`my:testKey[0]=true`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[true]`),
},
},
{
Array: []string{
`my:testKey[0]=10`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[10]`),
},
},
{
Array: []string{
`my:testKey[0]=-1`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "testKey"): config.NewObjectValue(`[-1]`),
},
},
{
Array: []string{
`my:names[0]=a`,
`my:names[1]=b`,
`my:names[2]=c`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "names"): config.NewObjectValue(`["a","b","c"]`),
},
},
{
Array: []string{
`my:names[0]=a`,
`my:names[1]=b`,
`my:names[2]=c`,
`my:names[0]=rewritten`,
},
Path: true,
Expected: config.Map{
config.MustMakeKey("my", "names"): config.NewObjectValue(`["rewritten","b","c"]`),
},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%v", test), func(t *testing.T) {
actual, err := parseConfig(test.Array, test.Path)
assert.NoError(t, err)
assert.Equal(t, test.Expected, actual)
})
}
}
func TestSetFail(t *testing.T) {
tests := []struct {
Array []string
Expected config.Map
}{
{
Array: []string{`my:[""]=value`},
},
{
Array: []string{"my:[0]=value"},
},
{
Array: []string{`my:name[-1]=value`},
},
{
Array: []string{`my:name[1]=value`},
},
{
Array: []string{`my:key.secure=value`},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%v", test), func(t *testing.T) {
_, err := parseConfig(test.Array, true /*path*/)
assert.Error(t, err)
})
}
}
const projectName = "test_project"
const stackName = "test_stack"

View file

@ -31,6 +31,7 @@ func newPreviewCmd() *cobra.Command {
var message string
var stack string
var configArray []string
var configPath bool
// Flags for engine.UpdateOptions.
var policyPackPaths []string
@ -95,7 +96,7 @@ func newPreviewCmd() *cobra.Command {
}
// Save any config values passed via flags.
if err := parseAndSaveConfigArray(s, configArray); err != nil {
if err := parseAndSaveConfigArray(s, configArray, configPath); err != nil {
return result.FromError(err)
}
@ -155,6 +156,9 @@ func newPreviewCmd() *cobra.Command {
cmd.PersistentFlags().StringArrayVarP(
&configArray, "config", "c", []string{},
"Config to use during the preview")
cmd.PersistentFlags().BoolVar(
&configPath, "config-path", false,
"Config keys contain a path to a property in a map or list to set")
cmd.PersistentFlags().StringVarP(
&message, "message", "m", "",

View file

@ -51,6 +51,7 @@ func newUpCmd() *cobra.Command {
var message string
var stack string
var configArray []string
var path bool
// Flags for engine.UpdateOptions.
var policyPackPaths []string
@ -78,7 +79,7 @@ func newUpCmd() *cobra.Command {
}
// Save any config values passed via flags.
if err := parseAndSaveConfigArray(s, configArray); err != nil {
if err := parseAndSaveConfigArray(s, configArray, path); err != nil {
return result.FromError(err)
}
@ -253,7 +254,7 @@ func newUpCmd() *cobra.Command {
}
// Prompt for config values (if needed) and save.
if err = handleConfig(s, templateNameOrURL, template, configArray, yes, opts.Display); err != nil {
if err = handleConfig(s, templateNameOrURL, template, configArray, yes, path, opts.Display); err != nil {
return result.FromError(err)
}
@ -379,6 +380,9 @@ func newUpCmd() *cobra.Command {
cmd.PersistentFlags().StringArrayVarP(
&configArray, "config", "c", []string{},
"Config to use during the update")
cmd.PersistentFlags().BoolVar(
&path, "config-path", false,
"Config keys contain a path to a property in a map or list to set")
cmd.PersistentFlags().StringVar(
&secretsProvider, "secrets-provider", "default", "The type of the provider that should be used to encrypt and "+
"decrypt secrets (possible choices: default, passphrase, awskms, azurekeyvault, gcpkms, hashivault). Only"+
@ -454,6 +458,7 @@ func handleConfig(
template workspace.Template,
configArray []string,
yes bool,
path bool,
opts display.Options) error {
// Get the existing config. stackConfig will be nil if there wasn't a previous deployment.
@ -480,7 +485,7 @@ func handleConfig(
// the stack's `pulumi:template` config value.
} else {
// Get config values passed on the command line.
commandLineConfig, parseErr := parseConfig(configArray)
commandLineConfig, parseErr := parseConfig(configArray, path)
if parseErr != nil {
return parseErr
}

View file

@ -315,11 +315,11 @@ func chooseStack(
// parseAndSaveConfigArray parses the config array and saves it as a config for
// the provided stack.
func parseAndSaveConfigArray(s backend.Stack, configArray []string) error {
func parseAndSaveConfigArray(s backend.Stack, configArray []string, path bool) error {
if len(configArray) == 0 {
return nil
}
commandLineConfig, err := parseConfig(configArray)
commandLineConfig, err := parseConfig(configArray, path)
if err != nil {
return err
}

View file

@ -1007,10 +1007,18 @@ func convertConfig(apiConfig map[string]apitype.ConfigValue) (config.Map, error)
if err != nil {
return nil, err
}
if rawV.Secret {
c[k] = config.NewSecureValue(rawV.String)
if rawV.Object {
if rawV.Secret {
c[k] = config.NewSecureObjectValue(rawV.String)
} else {
c[k] = config.NewObjectValue(rawV.String)
}
} else {
c[k] = config.NewValue(rawV.String)
if rawV.Secret {
c[k] = config.NewSecureValue(rawV.String)
} else {
c[k] = config.NewValue(rawV.String)
}
}
}
return c, nil

View file

@ -224,10 +224,18 @@ func (pc *Client) GetLatestConfiguration(ctx context.Context, stackID StackIdent
if err != nil {
return nil, err
}
if v.Secret {
cfg[newKey] = config.NewSecureValue(v.String)
if v.Object {
if v.Secret {
cfg[newKey] = config.NewSecureObjectValue(v.String)
} else {
cfg[newKey] = config.NewObjectValue(v.String)
}
} else {
cfg[newKey] = config.NewValue(v.String)
if v.Secret {
cfg[newKey] = config.NewSecureValue(v.String)
} else {
cfg[newKey] = config.NewValue(v.String)
}
}
}
@ -387,6 +395,7 @@ func (pc *Client) CreateUpdate(
wireConfig[k.String()] = apitype.ConfigValue{
String: v,
Secret: cv.Secure(),
Object: cv.Object(),
}
}

View file

@ -175,14 +175,14 @@ func makeEventEmitter(events chan<- Event, update UpdateInfo) (eventEmitter, err
continue
}
secret, err := v.Value(target.Decrypter)
secureValues, err := v.SecureValues(target.Decrypter)
if err != nil {
return eventEmitter{}, DecryptError{
Key: k,
Err: err,
}
}
secrets = append(secrets, secret)
secrets = append(secrets, secureValues...)
}
}

View file

@ -38,7 +38,12 @@ import (
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/httputil"
"github.com/pulumi/pulumi/pkg/workspace"
)
const (
// BookkeepingDir is the name of our bookkeeping folder, we store state here (like .git for git).
// Copied from workspace.BookkeepingDir to break import cycle.
BookkeepingDir = ".pulumi"
)
// Asset is a serialized asset reference. It is a union: thus, only one of its fields will be non-nil. Several helper
@ -828,7 +833,7 @@ func (a *Archive) readPath() (ArchiveReader, error) {
// If this is a .pulumi directory, we will skip this by default.
// TODO[pulumi/pulumi#122]: when we support .pulumiignore, this will be customizable.
if f.Name() == workspace.BookkeepingDir {
if f.Name() == BookkeepingDir {
if f.IsDir() {
return filepath.SkipDir
}

View file

@ -58,6 +58,36 @@ func (nopCrypter) EncryptValue(plaintext string) (string, error) {
return plaintext, nil
}
// TrackingDecrypter is a Decrypter that keeps track if decrypted values, which
// can be retrieved via SecureValues().
type TrackingDecrypter interface {
Decrypter
SecureValues() []string
}
// NewTrackingDecrypter returns a Decrypter that keeps track of decrypted values.
func NewTrackingDecrypter(decrypter Decrypter) TrackingDecrypter {
return &trackingDecrypter{decrypter: decrypter}
}
type trackingDecrypter struct {
decrypter Decrypter
secureValues []string
}
func (t *trackingDecrypter) DecryptValue(ciphertext string) (string, error) {
v, err := t.decrypter.DecryptValue(ciphertext)
if err != nil {
return "", err
}
t.secureValues = append(t.secureValues, v)
return v, nil
}
func (t *trackingDecrypter) SecureValues() []string {
return t.secureValues
}
// NewBlindingDecrypter returns a Decrypter that instead of decrypting data, just returns "[secret]", it can
// be used when you want to display configuration information to a user but don't want to prompt for a password
// so secrets will not be decrypted.

View file

@ -16,10 +16,18 @@ package config
import (
"encoding/json"
"fmt"
"strconv"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/resource"
)
var errSecureKeyReserved = errors.New(`"secure" key in maps of length 1 are reserved`)
// Map is a bag of config stored in the settings file.
type Map map[Key]Value
@ -47,6 +55,265 @@ func (m Map) HasSecureValue() bool {
return false
}
// Get gets the value for a given key. If path is true, the key's name portion is treated as a path.
func (m Map) Get(k Key, path bool) (Value, bool, error) {
// If the key isn't a path, go ahead and lookup the value.
if !path {
v, ok := m[k]
return v, ok, nil
}
// Otherwise, parse the path and get the new config key.
p, configKey, err := parseKeyPath(k)
if err != nil {
return Value{}, false, err
}
// If we only have a single path segment, go ahead and lookup the value.
if len(p) == 1 {
v, ok := m[configKey]
return v, ok, nil
}
// Otherwise, lookup the current root value and save it into a temporary map.
root := make(map[string]interface{})
if val, ok := m[configKey]; ok {
obj, err := val.ToObject()
if err != nil {
return Value{}, false, err
}
root[configKey.Name()] = obj
}
// Get the value within the object.
_, v, ok := getValueForPath(root, p)
if !ok {
return Value{}, false, nil
}
// If the value is a secure value, return it as one.
if is, s := isSecureValue(v); is {
return NewSecureValue(s), true, nil
}
// If it's a simple type, return it as a regular value.
switch t := v.(type) {
case string:
return NewValue(t), true, nil
case bool, int, uint, int32, uint32, int64, uint64, float32, float64:
return NewValue(fmt.Sprintf("%v", v)), true, nil
}
// Otherwise, return it as an object value.
json, err := json.Marshal(v)
if err != nil {
return Value{}, false, err
}
if hasSecureValue(v) {
return NewSecureObjectValue(string(json)), true, nil
}
return NewObjectValue(string(json)), true, nil
}
// Remove removes the value for a given key. If path is true, the key's name portion is treated as a path.
func (m Map) Remove(k Key, path bool) error {
// If the key isn't a path, go ahead and delete it and return.
if !path {
delete(m, k)
return nil
}
// Parse the path.
p, err := resource.ParsePropertyPath(k.Name())
if err != nil {
return errors.Wrap(err, "invalid config key path")
}
if len(p) == 0 {
return nil
}
firstKey, ok := p[0].(string)
if !ok || firstKey == "" {
return nil
}
configKey := MustMakeKey(k.Namespace(), firstKey)
// If we only have a single path segment, delete the key and return.
if len(p) == 1 {
delete(m, configKey)
return nil
}
// Otherwise, lookup the current root value and save it into a temporary map.
root := make(map[string]interface{})
if val, ok := m[configKey]; ok {
obj, err := val.ToObject()
if err != nil {
return err
}
root[configKey.Name()] = obj
}
// Get the value within the object up to the second-to-last path segment.
// If not found, exit early.
parent, dest, ok := getValueForPath(root, p[:len(p)-1])
if !ok {
return nil
}
// Remove the last path segment.
key := p[len(p)-1]
switch t := dest.(type) {
case []interface{}:
index, ok := key.(int)
if !ok || index < 0 || index >= len(t) {
return nil
}
t = append(t[:index], t[index+1:]...)
// Since we changed the array, we need to update the parent.
if parent != nil {
pkey := p[len(p)-2]
if _, err := setValue(parent, pkey, t, nil, nil); err != nil {
return err
}
}
case map[string]interface{}:
k, ok := key.(string)
if !ok {
return nil
}
delete(t, k)
// Secure values are reserved, so return an error when attempting to add one.
if isSecure, _ := isSecureValue(t); isSecure {
return errSecureKeyReserved
}
}
// Now, marshal then unmarshal the value, which will handle detecting
// whether it's a secure object or not.
jsonBytes, err := json.Marshal(root[configKey.Name()])
if err != nil {
return err
}
var v Value
if err = json.Unmarshal(jsonBytes, &v); err != nil {
return err
}
m[configKey] = v
return nil
}
// Set sets the value for a given key. If path is true, the key's name portion is treated as a path.
func (m Map) Set(k Key, v Value, path bool) error {
// If the key isn't a path, go ahead and set the value and return.
if !path {
m[k] = v
return nil
}
// Otherwise, parse the path and get the new config key.
p, configKey, err := parseKeyPath(k)
if err != nil {
return err
}
// If we only have a single path segment, set the value and return.
if len(p) == 1 {
m[configKey] = v
return nil
}
// Otherwise, lookup the current value and save it into a temporary map.
root := make(map[string]interface{})
if val, ok := m[configKey]; ok {
obj, err := val.ToObject()
if err != nil {
return err
}
// If obj is a string, set it to nil, which allows overwriting the existing
// top-level string value in the first iteration of the loop below.
if _, ok := obj.(string); ok {
obj = nil
}
root[configKey.Name()] = obj
}
// Now, loop through the path segments, and walk the object tree.
// If the value for a given segment is nil, create a new array/map.
// The root map is the initial cursor value, and parent is nil.
var parent interface{}
var parentKey interface{}
var cursor interface{}
var cursorKey interface{}
cursor = root
cursorKey = p[0]
for _, pkey := range p[1:] {
pvalue, err := getValue(cursor, cursorKey)
if err != nil {
return err
}
// If the value is nil, create a new array/map.
// Otherwise, return an error due to the type mismatch.
var newValue interface{}
switch pkey.(type) {
case int:
if pvalue == nil {
newValue = make([]interface{}, 0)
} else if _, ok := pvalue.([]interface{}); !ok {
return errors.Errorf("an array was expected for index %v", pkey)
}
case string:
if pvalue == nil {
newValue = make(map[string]interface{})
} else if _, ok := pvalue.(map[string]interface{}); !ok {
return errors.Errorf("a map was expected for key %q", pkey)
}
default:
contract.Failf("unexpected path type")
}
if newValue != nil {
pvalue = newValue
cursor, err = setValue(cursor, cursorKey, pvalue, parent, parentKey)
if err != nil {
return err
}
}
parent = cursor
parentKey = cursorKey
cursor = pvalue
cursorKey = pkey
}
// Adjust the value (e.g. convert "true"/"false" to booleans and integers to ints) and set it.
adjustedValue := adjustObjectValue(v, path)
if _, err = setValue(cursor, cursorKey, adjustedValue, parent, parentKey); err != nil {
return err
}
// Secure values are reserved, so return an error when attempting to add one.
if isSecure, _ := isSecureValue(cursor); isSecure {
return errSecureKeyReserved
}
// Serialize the updated object as JSON, and save it in the config map.
json, err := json.Marshal(root[configKey.Name()])
if err != nil {
return err
}
if v.Secure() {
m[configKey] = NewSecureObjectValue(string(json))
} else {
m[configKey] = NewObjectValue(string(json))
}
return nil
}
func (m Map) MarshalJSON() ([]byte, error) {
rawMap := make(map[string]Value, len(m))
for k, v := range m {
@ -104,3 +371,153 @@ func (m *Map) UnmarshalYAML(unmarshal func(interface{}) error) error {
*m = newMap
return nil
}
// parseKeyPath returns the property paths in the key and a new config key with the first
// path segment as the name.
func parseKeyPath(k Key) (resource.PropertyPath, Key, error) {
// Parse the path, which will be in the name portion of the key.
p, err := resource.ParsePropertyPath(k.Name())
if err != nil {
return nil, Key{}, errors.Wrap(err, "invalid config key path")
}
if len(p) == 0 {
return nil, Key{}, errors.New("empty config key path")
}
// Create a new key that has the first path segment as the name.
firstKey, ok := p[0].(string)
if !ok {
return nil, Key{}, errors.New("first path segement of config key must be a string")
}
if firstKey == "" {
return nil, Key{}, errors.New("config key is empty")
}
configKey := MustMakeKey(k.Namespace(), firstKey)
return p, configKey, nil
}
// getValueForPath returns the parent, value, and true if the value is found in source given the path segments in p.
func getValueForPath(source interface{}, p resource.PropertyPath) (interface{}, interface{}, bool) {
// If the source is nil, exit early.
if source == nil {
return nil, nil, false
}
// Lookup the value by each path segment.
var parent interface{}
v := source
for _, key := range p {
parent = v
switch t := v.(type) {
case []interface{}:
index, ok := key.(int)
if !ok || index < 0 || index >= len(t) {
return nil, nil, false
}
v = t[index]
case map[string]interface{}:
k, ok := key.(string)
if !ok {
return nil, nil, false
}
v, ok = t[k]
if !ok {
return nil, nil, false
}
default:
return nil, nil, false
}
}
return parent, v, true
}
// getValue returns the value in the container for the given key.
func getValue(container, key interface{}) (interface{}, error) {
switch t := container.(type) {
case []interface{}:
i, ok := key.(int)
contract.Assertf(ok, "key for an array must be an int")
// We explicitly allow i == len(t) here, which indicates a
// value that will be appended to the end of the array.
if i < 0 || i > len(t) {
return nil, errors.New("array index out of range")
}
if i == len(t) {
return nil, nil
}
return t[i], nil
case map[string]interface{}:
k, ok := key.(string)
contract.Assertf(ok, "key for a map must be a string")
return t[k], nil
}
contract.Failf("should not reach here")
return nil, nil
}
// Set value sets the value in the container for the given key, and returns the container.
// If the container is an array, and a value is being appended, containerParent and containerParentKey must
// not be nil since a new slice will be created to append the value, which needs to be saved in the parent.
// In this case, the new slice will be returned.
func setValue(container, key, value, containerParent, containerParentKey interface{}) (interface{}, error) {
switch t := container.(type) {
case []interface{}:
i, ok := key.(int)
contract.Assertf(ok, "key for an array must be an int")
// We allow i == len(t), which indicates the value should be appended to the end of the array.
if i < 0 || i > len(t) {
return nil, errors.New("array index out of range")
}
// If i == len(t), we need to append to the end of the array, which involves creating a new slice
// and saving it in the parent container.
if i == len(t) {
t = append(t, value)
contract.Assertf(containerParent != nil, "parent must not be nil")
contract.Assertf(containerParentKey != nil, "parentKey must not be nil")
if _, err := setValue(containerParent, containerParentKey, t, nil, nil); err != nil {
return nil, err
}
return t, nil
}
t[i] = value
case map[string]interface{}:
k, ok := key.(string)
contract.Assertf(ok, "key for a map must be a string")
t[k] = value
}
return container, nil
}
// adjustObjectValue returns a more suitable value for objects:
func adjustObjectValue(v Value, path bool) interface{} {
contract.Assertf(!v.Object(), "v must not be an Object")
// If the path flag isn't set, just return the value as-is.
if !path {
return v
}
// If it's a secure value, return as-is.
if v.Secure() {
return v
}
// If "false" or "true", return the boolean value.
if v.value == "false" {
return false
} else if v.value == "true" {
return true
}
// If it's convertible to an int, return the int.
i, err := strconv.Atoi(v.value)
if err == nil {
return i
}
// Otherwise, just return the string value.
return v.value
}

File diff suppressed because it is too large Load diff

View file

@ -16,13 +16,16 @@ package config
import (
"encoding/json"
"errors"
"fmt"
"github.com/pkg/errors"
)
// Value is a single config value.
type Value struct {
value string
secure bool
object bool
}
func NewSecureValue(v string) Value {
@ -33,6 +36,14 @@ func NewValue(v string) Value {
return Value{value: v, secure: false}
}
func NewSecureObjectValue(v string) Value {
return Value{value: v, secure: true, object: true}
}
func NewObjectValue(v string) Value {
return Value{value: v, secure: false, object: true}
}
// Value fetches the value of this configuration entry, using decrypter to decrypt if necessary. If the value
// is a secret and decrypter is nil, or if decryption fails for any reason, a non-nil error is returned.
func (c Value) Value(decrypter Decrypter) (string, error) {
@ -42,47 +53,126 @@ func (c Value) Value(decrypter Decrypter) (string, error) {
if decrypter == nil {
return "", errors.New("non-nil decrypter required for secret")
}
if c.object && decrypter != NopDecrypter {
var obj interface{}
if err := json.Unmarshal([]byte(c.value), &obj); err != nil {
return "", err
}
decryptedObj, err := decryptObject(obj, decrypter)
if err != nil {
return "", err
}
json, err := json.Marshal(decryptedObj)
if err != nil {
return "", err
}
return string(json), nil
}
return decrypter.DecryptValue(c.value)
}
func (c Value) SecureValues(decrypter Decrypter) ([]string, error) {
d := NewTrackingDecrypter(decrypter)
if _, err := c.Value(d); err != nil {
return nil, err
}
return d.SecureValues(), nil
}
func (c Value) Secure() bool {
return c.secure
}
func (c Value) MarshalJSON() ([]byte, error) {
if !c.secure {
return json.Marshal(c.value)
func (c Value) Object() bool {
return c.object
}
// ToObject returns the string value (if not an object), or the unmarshalled JSON object (if an object).
func (c Value) ToObject() (interface{}, error) {
if !c.object {
return c.value, nil
}
m := make(map[string]string)
m["secure"] = c.value
var v interface{}
err := json.Unmarshal([]byte(c.value), &v)
if err != nil {
return nil, err
}
return json.Marshal(m)
return v, nil
}
func (c Value) MarshalJSON() ([]byte, error) {
v, err := c.marshalValue()
if err != nil {
return nil, err
}
return json.Marshal(v)
}
func (c *Value) UnmarshalJSON(b []byte) error {
var m map[string]string
err := json.Unmarshal(b, &m)
if err == nil {
if len(m) != 1 {
return errors.New("malformed secure data")
}
val, has := m["secure"]
if !has {
return errors.New("malformed secure data")
}
c.value = val
c.secure = true
return nil
}
return json.Unmarshal(b, &c.value)
return c.unmarshalValue(
func(v interface{}) error {
return json.Unmarshal(b, v)
},
func(v interface{}) interface{} {
return v
})
}
func (c Value) MarshalYAML() (interface{}, error) {
return c.marshalValue()
}
func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
return c.unmarshalValue(func(v interface{}) error {
return unmarshal(v)
}, interfaceMapToStringMap)
}
func (c *Value) unmarshalValue(unmarshal func(interface{}) error, fix func(interface{}) interface{}) error {
// First, try to unmarshal as a string.
err := unmarshal(&c.value)
if err == nil {
c.secure = false
c.object = false
return nil
}
// Otherwise, try to unmarshal as an object.
var obj interface{}
if err = unmarshal(&obj); err != nil {
return errors.Wrapf(err, "malformed config value")
}
// Fix-up the object (e.g. convert `map[interface{}]interface{}` to `map[string]interface{}`).
obj = fix(obj)
if is, val := isSecureValue(obj); is {
c.value = val
c.secure = true
c.object = false
return nil
}
json, err := json.Marshal(obj)
if err != nil {
return errors.Wrapf(err, "marshalling obj")
}
c.value = string(json)
c.secure = hasSecureValue(obj)
c.object = true
return nil
}
func (c Value) marshalValue() (interface{}, error) {
if c.object {
var obj interface{}
err := json.Unmarshal([]byte(c.value), &obj)
return obj, err
}
if !c.secure {
return c.value, nil
}
@ -93,24 +183,93 @@ func (c Value) MarshalYAML() (interface{}, error) {
return m, nil
}
func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
var m map[string]string
err := unmarshal(&m)
if err == nil {
if len(m) != 1 {
return errors.New("malformed secure data")
// The unserialized value from YAML needs to be serializable as JSON, but YAML will unmarshal maps as
// `map[interface{}]interface{}` (because it supports bools as keys), which isn't supported by the JSON
// marshaller. To address, when unserializing YAML, we convert `map[interface{}]interface{}` to
// `map[string]interface{}`.
func interfaceMapToStringMap(v interface{}) interface{} {
switch t := v.(type) {
case map[interface{}]interface{}:
m := make(map[string]interface{})
for key, val := range t {
m[fmt.Sprintf("%v", key)] = interfaceMapToStringMap(val)
}
val, has := m["secure"]
if !has {
return errors.New("malformed secure data")
return m
case []interface{}:
a := make([]interface{}, len(t))
for i, val := range t {
a[i] = interfaceMapToStringMap(val)
}
return a
}
return v
}
c.value = val
c.secure = true
return nil
// hasSecureValue returns true if the object contains a value that's a `map[string]string` of
// length one with a "secure" key.
func hasSecureValue(v interface{}) bool {
switch t := v.(type) {
case map[string]interface{}:
if is, _ := isSecureValue(t); is {
return true
}
for _, val := range t {
if hasSecureValue(val) {
return true
}
}
case []interface{}:
for _, val := range t {
if hasSecureValue(val) {
return true
}
}
}
return false
}
// isSecureValue returns true if the object is a `map[string]string` of length one with a "secure" key.
func isSecureValue(v interface{}) (bool, string) {
if m, isMap := v.(map[string]interface{}); isMap && len(m) == 1 {
if val, hasSecureKey := m["secure"]; hasSecureKey {
if valString, isString := val.(string); isString {
return true, valString
}
}
}
return false, ""
}
// decryptObject returns a new object with all secure values in the object converted to decrypted strings.
func decryptObject(v interface{}, decrypter Decrypter) (interface{}, error) {
decryptIt := func(val interface{}) (interface{}, error) {
if isSecure, secureVal := isSecureValue(val); isSecure {
return decrypter.DecryptValue(secureVal)
}
return decryptObject(val, decrypter)
}
c.secure = false
return unmarshal(&c.value)
switch t := v.(type) {
case map[string]interface{}:
m := make(map[string]interface{})
for key, val := range t {
decrypted, err := decryptIt(val)
if err != nil {
return nil, err
}
m[key] = decrypted
}
return m, nil
case []interface{}:
a := make([]interface{}, len(t))
for i, val := range t {
decrypted, err := decryptIt(val)
if err != nil {
return nil, err
}
a[i] = decrypted
}
return a, nil
}
return v, nil
}

View file

@ -16,6 +16,7 @@ package config
import (
"encoding/json"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
@ -70,6 +71,180 @@ func TestMarshallSecureValueJSON(t *testing.T) {
assert.Equal(t, v, newV)
}
func TestHasSecureValue(t *testing.T) {
tests := []struct {
Value interface{}
Expected bool
}{
{
Value: []interface{}{"a", "b", "c"},
Expected: false,
},
{
Value: map[string]interface{}{
"foo": "bar",
"hi": map[string]interface{}{"secure": "securevalue", "but": "not"},
},
Expected: false,
},
{
Value: map[string]interface{}{
"foo": "bar",
"hi": map[string]interface{}{"secure": 1},
},
Expected: false,
},
{
Value: []interface{}{"a", "b", map[string]interface{}{"secure": "securevalue"}},
Expected: true,
},
{
Value: map[string]interface{}{
"foo": "bar",
"hi": map[string]interface{}{"secure": "securevalue"},
},
Expected: true,
},
{
Value: map[string]interface{}{
"foo": "bar",
"array": []interface{}{"a", "b", map[string]interface{}{"secure": "securevalue"}},
},
Expected: true,
},
{
Value: map[string]interface{}{
"foo": "bar",
"map": map[string]interface{}{
"nest": "blah",
"hi": map[string]interface{}{"secure": "securevalue"},
},
},
Expected: true,
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) {
jsonBytes, err := json.Marshal(test.Value)
assert.NoError(t, err)
var val interface{}
err = json.Unmarshal(jsonBytes, &val)
assert.NoError(t, err)
assert.Equal(t, test.Expected, hasSecureValue(val))
})
}
}
func TestDecryptingValue(t *testing.T) {
tests := []struct {
Value Value
Expected string
}{
{
Value: NewValue("value"),
Expected: "value",
},
{
Value: NewValue(`{"foo":"bar"}`),
Expected: `{"foo":"bar"}`,
},
{
Value: NewValue(`["a","b"]`),
Expected: `["a","b"]`,
},
{
Value: NewObjectValue(`{"foo":"bar"}`),
Expected: `{"foo":"bar"}`,
},
{
Value: NewObjectValue(`["a","b"]`),
Expected: `["a","b"]`,
},
{
Value: NewSecureValue("securevalue"),
Expected: "[secret]",
},
{
Value: NewSecureObjectValue(`{"foo":{"secure":"securevalue"}}`),
Expected: `{"foo":"[secret]"}`,
},
{
Value: NewSecureObjectValue(`["a",{"secure":"securevalue"}]`),
Expected: `["a","[secret]"]`,
},
}
decrypter := NewBlindingDecrypter()
for _, test := range tests {
t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) {
actual, err := test.Value.Value(decrypter)
assert.NoError(t, err)
assert.Equal(t, test.Expected, actual)
// Ensure the same value is returned when the NopDecrypter is used.
actualNop, err := test.Value.Value(NopDecrypter)
assert.NoError(t, err)
assert.Equal(t, test.Value.value, actualNop)
})
}
}
type passThroughDecrypter struct{}
func (d passThroughDecrypter) DecryptValue(ciphertext string) (string, error) {
return ciphertext, nil
}
func TestSecureValues(t *testing.T) {
tests := []struct {
Value Value
Expected []string
}{
{
Value: NewValue("value"),
Expected: nil,
},
{
Value: NewObjectValue(`{"foo":"bar"}`),
Expected: nil,
},
{
Value: NewObjectValue(`["a","b"]`),
Expected: nil,
},
{
Value: NewSecureValue("securevalue"),
Expected: []string{"securevalue"},
},
{
Value: NewSecureObjectValue(`{"foo":{"secure":"securevalue"}}`),
Expected: []string{"securevalue"},
},
{
Value: NewSecureObjectValue(`["a",{"secure":"securevalue"}]`),
Expected: []string{"securevalue"},
},
{
Value: NewSecureObjectValue(`["a",{"secure":"alpha"},{"test":{"secure":"beta"}}]`),
Expected: []string{"alpha", "beta"},
},
}
decrypter := passThroughDecrypter{}
for _, test := range tests {
t.Run(fmt.Sprintf("%v", test.Value), func(t *testing.T) {
actual, err := test.Value.SecureValues(decrypter)
assert.NoError(t, err)
assert.Equal(t, test.Expected, actual)
})
}
}
func roundtripValueYAML(v Value) (Value, error) {
return roundtripValue(v, yaml.Marshal, yaml.Unmarshal)
}

View file

@ -124,6 +124,18 @@ type TestStatsReporter interface {
ReportCommand(stats TestCommandStats)
}
// ConfigValue is used to provide config values to a test program.
type ConfigValue struct {
// The config key to pass to `pulumi config`.
Key string
// The config value to pass to `pulumi config`.
Value string
// Secret indicates that the `--secret` flag should be specified when calling `pulumi config`.
Secret bool
// Path indicates that the `--path` flag should be specified when calling `pulumi config`.
Path bool
}
// ProgramTestOptions provides options for ProgramTest
type ProgramTestOptions struct {
// Dir is the program directory to test.
@ -133,10 +145,12 @@ type ProgramTestOptions struct {
// Map of package names to versions. The test will use the specified versions of these packages instead of what
// is declared in `package.json`.
Overrides map[string]string
// Map of config keys and values to set (e.g. {"aws:region": "us-east-2"})
// Map of config keys and values to set (e.g. {"aws:region": "us-east-2"}).
Config map[string]string
// Map of secure config keys and values to set on the stack (e.g. {"aws:region": "us-east-2"})
// Map of secure config keys and values to set (e.g. {"aws:region": "us-east-2"}).
Secrets map[string]string
// List of config keys and values to set in order, including Secret and Path options.
OrderedConfig []ConfigValue
// SecretsProvider is the optional custom secrets provider to use instead of the default.
SecretsProvider string
// EditDirs is an optional list of edits to apply to the example, as subsequent deployments.
@ -923,6 +937,19 @@ func (pt *programTester) testLifeCycleInitialize(dir string) error {
}
}
for _, cv := range pt.opts.OrderedConfig {
configArgs := []string{"config", "set", cv.Key, cv.Value}
if cv.Secret {
configArgs = append(configArgs, "--secret")
}
if cv.Path {
configArgs = append(configArgs, "--path")
}
if err := pt.runPulumiCommand("pulumi-config", configArgs, dir); err != nil {
return err
}
}
return nil
}

View file

@ -31,7 +31,7 @@ import (
const (
// BackupDir is the name of the folder where backup stack information is stored.
BackupDir = "backups"
// BookkeepingDir is the name of our bookeeping folder, we store state here (like .git for git).
// BookkeepingDir is the name of our bookkeeping folder, we store state here (like .git for git).
BookkeepingDir = ".pulumi"
// ConfigDir is the name of the folder that holds local configuration information.
ConfigDir = "config"

View file

@ -1,6 +1,8 @@
// Copyright 2016-2019, Pulumi Corporation. All rights reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Pulumi;
@ -8,23 +10,136 @@ class Program
{
static Task<int> Main(string[] args)
{
return Deployment.RunAsync(() =>
return Deployment.RunAsync(() =>
{
var config = new Config("config_basic_dotnet");
// This value is plaintext and doesn't require encryption.
var value = config.Require("aConfigValue");
if (value != "this value is a value")
var tests = new[]
{
throw new Exception($"aConfigValue not the expected value; got {value}");
}
new Test
{
Key = "aConfigValue",
Expected = "this value is a value"
},
new Test
{
Key = "bEncryptedSecret",
Expected = "this super secret is encrypted"
},
new Test
{
Key = "outer",
Expected = "{\"inner\":\"value\"}",
AdditionalValidation = () =>
{
var outer = config.RequireObject<Dictionary<string, string>>("outer");
if (outer.Count != 1 || outer["inner"] != "value")
{
throw new Exception("'outer' not the expected object value");
}
}
},
new Test
{
Key = "names",
Expected = "[\"a\",\"b\",\"c\",\"super secret name\"]",
AdditionalValidation = () =>
{
var expected = new[] { "a", "b", "c", "super secret name" };
var names = config.RequireObject<string[]>("names");
if (!Enumerable.SequenceEqual(expected, names))
{
throw new Exception("'names' not the expected object value");
}
}
},
new Test
{
Key = "servers",
Expected = "[{\"host\":\"example\",\"port\":80}]",
AdditionalValidation = () =>
{
var servers = config.RequireObject<Server[]>("servers");
if (servers.Length != 1 || servers[0].host != "example" || servers[0].port != 80)
{
throw new Exception("'servers' not the expected object value");
}
}
},
new Test
{
Key = "a",
Expected = "{\"b\":[{\"c\":true},{\"c\":false}]}",
AdditionalValidation = () =>
{
var a = config.RequireObject<A>("a");
if (a.b.Length != 2 || a.b[0].c != true || a.b[1].c != false)
{
throw new Exception("'a' not the expected object value");
}
}
},
new Test
{
Key = "tokens",
Expected = "[\"shh\"]",
AdditionalValidation = () =>
{
var expected = new[] { "shh" };
var tokens = config.RequireObject<string[]>("tokens");
if (!Enumerable.SequenceEqual(expected, tokens))
{
throw new Exception("'tokens' not the expected object value");
}
}
},
new Test
{
Key = "foo",
Expected = "{\"bar\":\"don't tell\"}",
AdditionalValidation = () =>
{
var foo = config.RequireObject<Dictionary<string, string>>("foo");
if (foo.Count != 1 || foo["bar"] != "don't tell")
{
throw new Exception("'foo' not the expected object value");
}
}
},
};
// This value is a secret
var secret = config.Require("bEncryptedSecret");
if (secret != "this super secret is encrypted")
foreach (var test in tests)
{
throw new Exception($"bEncryptedSecret not the expected value; got {secret}");
var value = config.Require(test.Key);
if (value != test.Expected)
{
throw new Exception($"'{test.Key}' not the expected value; got {value}");
}
test.AdditionalValidation?.Invoke();
}
});
}
}
}
class Test
{
public string Key;
public string Expected;
public Action AdditionalValidation;
}
class Server
{
public string host { get; set; }
public int port { get; set; }
}
class A
{
public B[] b { get; set; }
}
class B
{
public bool c { get; set; }
}

View file

@ -4,6 +4,7 @@ package main
import (
"fmt"
"github.com/pulumi/pulumi/sdk/go/pulumi"
"github.com/pulumi/pulumi/sdk/go/pulumi/config"
)
@ -13,16 +14,49 @@ func main() {
// Just test that basic config works.
cfg := config.New(ctx, "config_basic_go")
// This value is plaintext and doesn't require encryption.
value := cfg.Require("aConfigValue")
if value != "this value is a value" {
return fmt.Errorf("aConfigValue not the expected value; got %s", value)
tests := []struct {
Key string
Expected string
}{
{
Key: "aConfigValue",
Expected: `this value is a value`,
},
{
Key: "bEncryptedSecret",
Expected: `this super secret is encrypted`,
},
{
Key: "outer",
Expected: `{"inner":"value"}`,
},
{
Key: "names",
Expected: `["a","b","c","super secret name"]`,
},
{
Key: "servers",
Expected: `[{"host":"example","port":80}]`,
},
{
Key: "a",
Expected: `{"b":[{"c":true},{"c":false}]}`,
},
{
Key: "tokens",
Expected: `["shh"]`,
},
{
Key: "foo",
Expected: `{"bar":"don't tell"}`,
},
}
// This value is a secret and is encrypted using the passphrase `supersecret`.
secret := cfg.Require("bEncryptedSecret")
if secret != "this super secret is encrypted" {
return fmt.Errorf("bEncryptedSecret not the expected value; got %s", secret)
for _, test := range tests {
value := cfg.Require(test.Key)
if value != test.Expected {
return fmt.Errorf("%q not the expected value; got %q", test.Key, value)
}
}
return nil

View file

@ -8,8 +8,52 @@ const config = new Config("config_basic_js");
// This value is plaintext and doesn't require encryption.
const value = config.require("aConfigValue");
assert.equal(value, "this value is a value", "aConfigValue not the expected value");
assert.strictEqual(value, "this value is a value", "'aConfigValue' not the expected value");
// This value is a secret and is encrypted using the passphrase `supersecret`.
const secret = config.require("bEncryptedSecret");
assert.equal(secret, "this super secret is encrypted", "bEncryptedSecret not the expected value");
assert.strictEqual(secret, "this super secret is encrypted", "'bEncryptedSecret' not the expected value");
const testData: {
key: string;
expectedJSON: string;
expectedObject: any;
}[] = [
{
key: "outer",
expectedJSON: `{"inner":"value"}`,
expectedObject: { inner: "value" },
},
{
key: "names",
expectedJSON: `["a","b","c","super secret name"]`,
expectedObject: ["a", "b", "c", "super secret name"],
},
{
key: "servers",
expectedJSON: `[{"host":"example","port":80}]`,
expectedObject: [{ host: "example", port: 80 }],
},
{
key: "a",
expectedJSON: `{"b":[{"c":true},{"c":false}]}`,
expectedObject: { b: [{ c: true }, { c: false }] },
},
{
key: "tokens",
expectedJSON: `["shh"]`,
expectedObject: ["shh"],
},
{
key: "foo",
expectedJSON: `{"bar":"don't tell"}`,
expectedObject: { bar: "don't tell" },
},
];
for (const test of testData) {
const json = config.require(test.key);
const obj = config.requireObject(test.key);
assert.strictEqual(json, test.expectedJSON, `'${test.key}' not the expected JSON`);
assert.deepStrictEqual(obj, test.expectedObject, `'${test.key}' not the expected object`);
}

View file

@ -12,3 +12,42 @@ assert value == 'this value is a Pythonic value'
# This value is a secret and is encrypted using the passphrase `supersecret`.
secret = config.require('bEncryptedSecret')
assert secret == 'this super Pythonic secret is encrypted'
test_data = [
{
'key': 'outer',
'expected_json': '{"inner":"value"}',
'expected_object': { 'inner': 'value' }
},
{
'key': 'names',
'expected_json': '["a","b","c","super secret name"]',
'expected_object': ['a', 'b', 'c', 'super secret name']
},
{
'key': 'servers',
'expected_json': '[{"host":"example","port":80}]',
'expected_object': [{ 'host': 'example', 'port': 80 }]
},
{
'key': 'a',
'expected_json': '{"b":[{"c":true},{"c":false}]}',
'expected_object': { 'b': [{ 'c': True }, { 'c': False }] }
},
{
'key': 'tokens',
'expected_json': '["shh"]',
'expected_object': ['shh']
},
{
'key': 'foo',
'expected_json': '{"bar":"don\'t tell"}',
'expected_object': { 'bar': "don't tell" }
}
]
for test in test_data:
json = config.require(test['key'])
obj = config.require_object(test['key'])
assert json == test['expected_json']
assert obj == test['expected_object']

View file

@ -562,6 +562,301 @@ func TestConfigSave(t *testing.T) {
e.RunCommand("pulumi", "stack", "rm", "--yes")
}
// TestConfigPaths ensures that config commands with paths work as expected.
func TestConfigPaths(t *testing.T) {
e := ptesting.NewEnvironment(t)
defer func() {
if !t.Failed() {
e.DeleteEnvironment()
}
}()
// Initialize an empty stack.
path := filepath.Join(e.RootPath, "Pulumi.yaml")
err := (&workspace.Project{
Name: "testing-config",
Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil),
}).Save(path)
assert.NoError(t, err)
e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
e.RunCommand("pulumi", "stack", "init", "testing")
namespaces := []string{"", "my:"}
tests := []struct {
Key string
Value string
Secret bool
Path bool
TopLevelKey string
TopLevelExpectedValue string
}{
{
Key: "aConfigValue",
Value: "this value is a value",
TopLevelKey: "aConfigValue",
TopLevelExpectedValue: "this value is a value",
},
{
Key: "anotherConfigValue",
Value: "this value is another value",
TopLevelKey: "anotherConfigValue",
TopLevelExpectedValue: "this value is another value",
},
{
Key: "bEncryptedSecret",
Value: "this super secret is encrypted",
Secret: true,
TopLevelKey: "bEncryptedSecret",
TopLevelExpectedValue: "this super secret is encrypted",
},
{
Key: "anotherEncryptedSecret",
Value: "another encrypted secret",
Secret: true,
TopLevelKey: "anotherEncryptedSecret",
TopLevelExpectedValue: "another encrypted secret",
},
{
Key: "[]",
Value: "square brackets value",
TopLevelKey: "[]",
TopLevelExpectedValue: "square brackets value",
},
{
Key: "x.y",
Value: "x.y value",
TopLevelKey: "x.y",
TopLevelExpectedValue: "x.y value",
},
{
Key: "0",
Value: "0 value",
Path: true,
TopLevelKey: "0",
TopLevelExpectedValue: "0 value",
},
{
Key: "true",
Value: "value",
Path: true,
TopLevelKey: "true",
TopLevelExpectedValue: "value",
},
{
Key: `["test.Key"]`,
Value: "test key value",
Path: true,
TopLevelKey: "test.Key",
TopLevelExpectedValue: "test key value",
},
{
Key: `nested["test.Key"]`,
Value: "nested test key value",
Path: true,
TopLevelKey: "nested",
TopLevelExpectedValue: `{"test.Key":"nested test key value"}`,
},
{
Key: "outer.inner",
Value: "value",
Path: true,
TopLevelKey: "outer",
TopLevelExpectedValue: `{"inner":"value"}`,
},
{
Key: "names[0]",
Value: "a",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a"]`,
},
{
Key: "names[1]",
Value: "b",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b"]`,
},
{
Key: "names[2]",
Value: "c",
Path: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b","c"]`,
},
{
Key: "names[3]",
Value: "super secret name",
Path: true,
Secret: true,
TopLevelKey: "names",
TopLevelExpectedValue: `["a","b","c","super secret name"]`,
},
{
Key: "servers[0].port",
Value: "80",
Path: true,
TopLevelKey: "servers",
TopLevelExpectedValue: `[{"port":80}]`,
},
{
Key: "servers[0].host",
Value: "example",
Path: true,
TopLevelKey: "servers",
TopLevelExpectedValue: `[{"host":"example","port":80}]`,
},
{
Key: "a.b[0].c",
Value: "true",
Path: true,
TopLevelKey: "a",
TopLevelExpectedValue: `{"b":[{"c":true}]}`,
},
{
Key: "a.b[1].c",
Value: "false",
Path: true,
TopLevelKey: "a",
TopLevelExpectedValue: `{"b":[{"c":true},{"c":false}]}`,
},
{
Key: "tokens[0]",
Value: "shh",
Path: true,
Secret: true,
TopLevelKey: "tokens",
TopLevelExpectedValue: `["shh"]`,
},
{
Key: "foo.bar",
Value: "don't tell",
Path: true,
Secret: true,
TopLevelKey: "foo",
TopLevelExpectedValue: `{"bar":"don't tell"}`,
},
{
Key: "semiInner.a.b.c.d",
Value: "1",
Path: true,
TopLevelKey: "semiInner",
TopLevelExpectedValue: `{"a":{"b":{"c":{"d":1}}}}`,
},
{
Key: "wayInner.a.b.c.d.e.f.g.h.i.j.k",
Value: "false",
Path: true,
TopLevelKey: "wayInner",
TopLevelExpectedValue: `{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":{"k":false}}}}}}}}}}}`,
},
// Overwriting a top-level string value is allowed.
{
Key: "aConfigValue.inner",
Value: "new value",
Path: true,
TopLevelKey: "aConfigValue",
TopLevelExpectedValue: `{"inner":"new value"}`,
},
{
Key: "anotherConfigValue[0]",
Value: "new value",
Path: true,
TopLevelKey: "anotherConfigValue",
TopLevelExpectedValue: `["new value"]`,
},
{
Key: "bEncryptedSecret.inner",
Value: "new value",
Path: true,
TopLevelKey: "bEncryptedSecret",
TopLevelExpectedValue: `{"inner":"new value"}`,
},
{
Key: "anotherEncryptedSecret[0]",
Value: "new value",
Path: true,
TopLevelKey: "anotherEncryptedSecret",
TopLevelExpectedValue: `["new value"]`,
},
}
validateConfigGet := func(key string, value string, path bool) {
args := []string{"config", "get", key}
if path {
args = append(args, "--path")
}
stdout, stderr := e.RunCommand("pulumi", args...)
assert.Equal(t, fmt.Sprintf("%s\n", value), stdout)
assert.Equal(t, "", stderr)
}
for _, ns := range namespaces {
for _, test := range tests {
key := fmt.Sprintf("%s%s", ns, test.Key)
topLevelKey := fmt.Sprintf("%s%s", ns, test.TopLevelKey)
// Set the value.
args := []string{"config", "set"}
if test.Secret {
args = append(args, "--secret")
}
if test.Path {
args = append(args, "--path")
}
args = append(args, key, test.Value)
stdout, stderr := e.RunCommand("pulumi", args...)
assert.Equal(t, "", stdout)
assert.Equal(t, "", stderr)
// Get the value and validate it.
validateConfigGet(key, test.Value, test.Path)
// Get the top-level value and validate it.
validateConfigGet(topLevelKey, test.TopLevelExpectedValue, false /*path*/)
}
}
badKeys := []string{
// Syntax errors.
"root[",
`root["nested]`,
"root.array[abc]",
"root.[1]",
// First path segment must be a non-empty string.
`[""]`,
"[0]",
// Index out of range.
"names[-1]",
"names[5]",
// A "secure" key that is a map with a single string value is reserved by the system.
"key.secure",
"super.nested.map.secure",
// Type mismatch.
"outer[0]",
"names.nested",
"outer.inner.nested",
"outer.inner[0]",
}
for _, ns := range namespaces {
for _, badKey := range badKeys {
key := fmt.Sprintf("%s%s", ns, badKey)
stdout, stderr := e.RunCommandExpectError("pulumi", "config", "set", "--path", key, "value")
assert.Equal(t, "", stdout)
assert.NotEqual(t, "", stderr)
}
}
e.RunCommand("pulumi", "stack", "rm", "--yes")
}
// Tests basic configuration from the perspective of a Pulumi program.
func TestConfigBasicNodeJS(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
@ -574,6 +869,19 @@ func TestConfigBasicNodeJS(t *testing.T) {
Secrets: map[string]string{
"bEncryptedSecret": "this super secret is encrypted",
},
OrderedConfig: []integration.ConfigValue{
{Key: "outer.inner", Value: "value", Path: true},
{Key: "names[0]", Value: "a", Path: true},
{Key: "names[1]", Value: "b", Path: true},
{Key: "names[2]", Value: "c", Path: true},
{Key: "names[3]", Value: "super secret name", Path: true, Secret: true},
{Key: "servers[0].port", Value: "80", Path: true},
{Key: "servers[0].host", Value: "example", Path: true},
{Key: "a.b[0].c", Value: "true", Path: true},
{Key: "a.b[1].c", Value: "false", Path: true},
{Key: "tokens[0]", Value: "shh", Path: true, Secret: true},
{Key: "foo.bar", Value: "don't tell", Path: true, Secret: true},
},
})
}
@ -599,9 +907,11 @@ func TestInvalidVersionInPackageJson(t *testing.T) {
// Tests basic configuration from the perspective of a Pulumi program.
func TestConfigBasicPython(t *testing.T) {
t.Skip("pulumi/pulumi#2138")
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: filepath.Join("config_basic", "python"),
Dir: filepath.Join("config_basic", "python"),
Dependencies: []string{
path.Join("..", "..", "sdk", "python", "env", "src"),
},
Quick: true,
Config: map[string]string{
"aConfigValue": "this value is a Pythonic value",
@ -609,6 +919,19 @@ func TestConfigBasicPython(t *testing.T) {
Secrets: map[string]string{
"bEncryptedSecret": "this super Pythonic secret is encrypted",
},
OrderedConfig: []integration.ConfigValue{
{Key: "outer.inner", Value: "value", Path: true},
{Key: "names[0]", Value: "a", Path: true},
{Key: "names[1]", Value: "b", Path: true},
{Key: "names[2]", Value: "c", Path: true},
{Key: "names[3]", Value: "super secret name", Path: true, Secret: true},
{Key: "servers[0].port", Value: "80", Path: true},
{Key: "servers[0].host", Value: "example", Path: true},
{Key: "a.b[0].c", Value: "true", Path: true},
{Key: "a.b[1].c", Value: "false", Path: true},
{Key: "tokens[0]", Value: "shh", Path: true, Secret: true},
{Key: "foo.bar", Value: "don't tell", Path: true, Secret: true},
},
})
}
@ -623,6 +946,19 @@ func TestConfigBasicGo(t *testing.T) {
Secrets: map[string]string{
"bEncryptedSecret": "this super secret is encrypted",
},
OrderedConfig: []integration.ConfigValue{
{Key: "outer.inner", Value: "value", Path: true},
{Key: "names[0]", Value: "a", Path: true},
{Key: "names[1]", Value: "b", Path: true},
{Key: "names[2]", Value: "c", Path: true},
{Key: "names[3]", Value: "super secret name", Path: true, Secret: true},
{Key: "servers[0].port", Value: "80", Path: true},
{Key: "servers[0].host", Value: "example", Path: true},
{Key: "a.b[0].c", Value: "true", Path: true},
{Key: "a.b[1].c", Value: "false", Path: true},
{Key: "tokens[0]", Value: "shh", Path: true, Secret: true},
{Key: "foo.bar", Value: "don't tell", Path: true, Secret: true},
},
})
}
@ -637,6 +973,19 @@ func TestConfigBasicDotNet(t *testing.T) {
Secrets: map[string]string{
"bEncryptedSecret": "this super secret is encrypted",
},
OrderedConfig: []integration.ConfigValue{
{Key: "outer.inner", Value: "value", Path: true},
{Key: "names[0]", Value: "a", Path: true},
{Key: "names[1]", Value: "b", Path: true},
{Key: "names[2]", Value: "c", Path: true},
{Key: "names[3]", Value: "super secret name", Path: true, Secret: true},
{Key: "servers[0].port", Value: "80", Path: true},
{Key: "servers[0].host", Value: "example", Path: true},
{Key: "a.b[0].c", Value: "true", Path: true},
{Key: "a.b[1].c", Value: "false", Path: true},
{Key: "tokens[0]", Value: "shh", Path: true, Secret: true},
{Key: "foo.bar", Value: "don't tell", Path: true, Secret: true},
},
})
}