maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Correctly push secretness up during deserialization in runtimes

Browse source 
There current RPC model for Pulumi allows secret values to be deeply
embedded in lists or maps, however at the language level, since we
track secrets via `Output<T>` we need to ensure that during
deserialization, if a list or a map contains a secret, we need to
instead treat it as if the entire list or map was a secret.

We have logic in the language runtimes to do this as part of
serialization. There were a few issues this commit addresses:

- We were not promoting secretness across arrays in either Node or
  Python
- For Python, our promotion logic was buggy and caused it to behave in
  a manner where if any value was secret, the output values of the
  object would be corrupted, because we'd incorrectly treat the
  outputs as a secret who's value was a map, instead of a map of
  values (some of which may be secret).

This caused very confusing behavior, because it would appear that a
resource creation call just did not set various output properties when
one or more of them ended up containing a secret.
This commit is contained in:
Matt Ellis 2019-07-02 16:31:14 -07:00
parent ed46891693
commit 858517a7ca
5 changed files with 162 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -6,6 +6,8 @@ CHANGELOG
[#2672](https://github.com/pulumi/pulumi/issues/2672))
- Fix an issue where a file archive created on Windows would contain back-slashes
[#2784](https://github.com/pulumi/pulumi/issues/2784))
- Fix an issue where output values of a resource would not be present when they
contained secret values, when using Python.
- Fix an issue where emojis are printed in non-interactive mode. (fixes
[#2871](https://github.com/pulumi/pulumi/issues/2871))

54
sdk/nodejs/runtime/rpc.ts
View file

@ -175,10 +175,8 @@ export function resolveProperties(
// If this value is a secret, unwrap its inner value.
let value = allProps[k];
const isSecret = value && value[specialSecretSig] === true;
if (isSecret) {
value = value.value;
}
const isSecret = isRpcSecret(value);
value = unwrapRpcSecret(value);
try {
// If either we are performing a real deployment, or this is a stable property value, we
@ -369,6 +367,23 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
}
}
/**
* isRpcSecret returns true if obj is a wrapped secret value (i.e. it's an object with the special key set).
*/
function isRpcSecret(obj: any): boolean {
return obj && obj[specialSigKey] === specialSecretSig;
}
/**
* unwrapRpcSecret returns the underlying value for a secret, or the value itself if it was not a secret.
*/
function unwrapRpcSecret(obj: any): any {
if (!isRpcSecret(obj)) {
return obj;
}
return obj.value;
}
/**
* deserializeProperty unpacks some special types, reversing the above process.
*/
@ -383,10 +398,24 @@ export function deserializeProperty(prop: any): any {
return prop;
}
else if (prop instanceof Array) {
// We can just deserialize all the elements of the underyling array and return it.
// However, we want to push secretness up to the top level (since we can't set sub-properties to secret)
// values since they are not typed as Output<T>.
let hadSecret = false;
const elems: any[] = [];
for (const e of prop) {
elems.push(deserializeProperty(e));
prop = deserializeProperty(e);
hadSecret = hadSecret || isRpcSecret(prop);
elems.push(unwrapRpcSecret(prop));
}
if (hadSecret) {
return {
[specialSigKey]: specialSecretSig,
value: elems,
};
}
return elems;
}
else {
@ -431,7 +460,7 @@ export function deserializeProperty(prop: any): any {
}
case specialSecretSig:
return {
[specialSecretSig]: true,
[specialSigKey]: specialSecretSig,
value: deserializeProperty(prop["value"]),
};
default:
@ -446,19 +475,14 @@ export function deserializeProperty(prop: any): any {
let hadSecrets = false;
for (const k of Object.keys(prop)) {
let o = deserializeProperty(prop[k]);
if (o && o[specialSecretSig] === true) {
hadSecrets = true;
o = o.value;
}
obj[k] = o;
const o = deserializeProperty(prop[k]);
hadSecrets = hadSecrets || isRpcSecret(o);
obj[k] = unwrapRpcSecret(o);
}
if (hadSecrets) {
return {
[specialSecretSig]: true,
[specialSigKey]: specialSecretSig,
value: obj,
};
}

50
sdk/nodejs/tests/runtime/props.spec.ts
View file

@ -16,6 +16,7 @@ import * as assert from "assert";
import { Inputs, runtime } from "../../index";
import { asyncTest } from "../util";
const gstruct = require("google-protobuf/google/protobuf/struct_pb.js");
describe("runtime", () => {
@ -53,5 +54,54 @@ describe("runtime", () => {
[runtime.specialSigKey]: "foobar",
}));
});
it("pushed secretness up correctly", () => {
const secretValue = {
[runtime.specialSigKey]: runtime.specialSecretSig,
"value": "a secret value",
};
const props = gstruct.Struct.fromJavaScript({
"regular": "a normal value",
"list": [ "a normal value", "another value", secretValue ],
"map": { "regular": "a normal value", "secret": secretValue },
"mapWithList": {
"regular": "a normal value",
"list": [ "a normal value", secretValue ],
},
"listWithMap": [{
"regular": "a normal value",
"secret": secretValue,
}],
});
const result = runtime.deserializeProperties(props);
console.log(JSON.stringify(result));
// Regular had no secrets in it, so it is returned as is.
assert.equal(result.regular, "a normal value");
// One of the elements in the list was a secret, so the secretness is promoted to top level.
assert.equal(result.list[runtime.specialSigKey], runtime.specialSecretSig);
assert.equal(result.list.value[0], "a normal value");
assert.equal(result.list.value[1], "another value");
assert.equal(result.list.value[2], "a secret value");
// One of the values of the map was a secret, so the secretness is promoted to top level.
assert.equal(result.map[runtime.specialSigKey], runtime.specialSecretSig);
assert.equal(result.map.value.regular, "a normal value");
assert.equal(result.map.value.secret, "a secret value");
// The nested map had a secret in one of the values, so the entire thing becomes a secret.
assert.equal(result.mapWithList[runtime.specialSigKey], runtime.specialSecretSig);
assert.equal(result.mapWithList.value.regular, "a normal value");
assert.equal(result.mapWithList.value.list[0], "a normal value");
assert.equal(result.mapWithList.value.list[1], "a secret value");
// An array element contained a secret (via a nested map), so the entrie array becomes a secret.
assert.equal(result.listWithMap[runtime.specialSigKey], runtime.specialSecretSig);
assert.equal(result.listWithMap.value[0].regular, "a normal value");
assert.equal(result.listWithMap.value[0].secret, "a secret value");
});
});
});

49
sdk/python/lib/pulumi/runtime/rpc.py
View file

@ -209,25 +209,28 @@ def deserialize_properties(props_struct: struct_pb2.Struct) -> Any:
# that if the struct had any secret properties, we push the secretness of the object up to us
# since we can only set secret outputs on top level properties.
output = {}
had_secret = False
for k, v in list(props_struct.items()):
value = deserialize_property(v)
# We treat values that deserialize to "None" as if they don't exist.
if value is not None:
if isinstance(value, dict) and _special_sig_key in value and value[_special_sig_key] == _special_secret_sig:
had_secret = True
value = value["value"]
output[k] = value
if had_secret:
return {
_special_sig_key: _special_secret_sig,
"value": output
}
return output
def is_rpc_secret(value: Any) -> bool:
"""
Returns if a given python value is actually a wrapped secret
"""
return isinstance(value, dict) and _special_sig_key in value and value[_special_sig_key] == _special_secret_sig
def unwrap_rpc_secret(value: Any) -> Any:
"""
Given a value, if it is a wrapped secret value, return the underlying, otherwise return the value unmodified.
"""
if is_rpc_secret(value):
return value["value"]
return value
def deserialize_property(value: Any) -> Any:
"""
@ -239,11 +242,31 @@ def deserialize_property(value: Any) -> Any:
# ListValues are projected to lists
if isinstance(value, struct_pb2.ListValue):
return [deserialize_property(v) for v in value]
values = [deserialize_property(v) for v in value]
# If there are any secret values in the list, push the secretness "up" a level by returning
# an array that is marked as a secret with raw values inside.
if any(is_rpc_secret(v) for v in values):
return {
_special_sig_key: _special_secret_sig,
"value": [unwrap_rpc_secret(v) for v in values]
}
return values
# Structs are projected to dictionaries
if isinstance(value, struct_pb2.Struct):
return deserialize_properties(value)
props = deserialize_properties(value)
# If there are any secret values in the dictionary, push the secretness "up" a level by returning
# a dictionary that is marked as a secret with raw values inside. Note: thje isinstance check here is
# important, since deserialize_properties will return either a dictionary or a concret type (in the case of
# assets).
if isinstance(props, dict) and any(is_rpc_secret(v) for v in props.values()):
return {
_special_sig_key: _special_secret_sig,
"value": {k: unwrap_rpc_secret(v) for k, v in props.items()}
}
return props
# Everything else is identity projected.
return value

35
sdk/python/lib/test/test_next_serialize.py
View file

@ -246,3 +246,38 @@ class DeserializationTests(unittest.TestCase):
except AssertionError as err:
error = err
self.assertIsNotNone(error)
def test_secret_push_up(self):
secret_value = {rpc._special_sig_key: rpc._special_secret_sig, "value": "a secret value" }
all_props = struct_pb2.Struct()
all_props["regular"] = "a normal value"
all_props["list"] = ["a normal value", "another value", secret_value]
all_props["map"] = {"regular": "a normal value", "secret": secret_value}
all_props["mapWithList"] = {"regular": "a normal value", "list": ["a normal value", secret_value]}
all_props["listWithMap"] = [{"regular": "a normal value", "secret": secret_value}]
val = rpc.deserialize_properties(all_props)
self.assertEqual(all_props["regular"], val["regular"])
self.assertIsInstance(val["list"], dict)
self.assertEqual(val["list"][rpc._special_sig_key], rpc._special_secret_sig)
self.assertEqual(val["list"]["value"][0], "a normal value")
self.assertEqual(val["list"]["value"][1], "another value")
self.assertEqual(val["list"]["value"][2], "a secret value")
self.assertIsInstance(val["map"], dict)
self.assertEqual(val["map"][rpc._special_sig_key], rpc._special_secret_sig)
self.assertEqual(val["map"]["value"]["regular"], "a normal value")
self.assertEqual(val["map"]["value"]["secret"], "a secret value")
self.assertIsInstance(val["mapWithList"], dict)
self.assertEqual(val["mapWithList"][rpc._special_sig_key], rpc._special_secret_sig)
self.assertEqual(val["mapWithList"]["value"]["regular"], "a normal value")
self.assertEqual(val["mapWithList"]["value"]["list"][0], "a normal value")
self.assertEqual(val["mapWithList"]["value"]["list"][1], "a secret value")
self.assertIsInstance(val["listWithMap"], dict)
self.assertEqual(val["listWithMap"][rpc._special_sig_key], rpc._special_secret_sig)
self.assertEqual(val["listWithMap"]["value"][0]["regular"], "a normal value")
self.assertEqual(val["listWithMap"]["value"][0]["secret"], "a secret value")