maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[sdk/nodejs] Marshal output values (#7925)

Browse source 
This change adds support for marshaling outputs as output values in the Node.js SDK.
This commit is contained in:
Justin Van Patten 2021-09-15 18:25:26 -07:00 committed by GitHub
parent 054b3f9edc
commit 0b9e41e8c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 170 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
sdk/nodejs/runtime/invoke.ts
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -255,7 +255,10 @@ export function call<T>(tok: string, props: Inputs, res?: Resource): Output<T> {
version = res.__version;
}
const [serialized, propertyDepsResources] = await serializePropertiesReturnDeps(`call:${tok}`, props);
// We keep output values when serializing inputs for call.
const [serialized, propertyDepsResources] = await serializePropertiesReturnDeps(`call:${tok}`, props, {
keepOutputValues: true,
});
log.debug(`Call RPC prepared: tok=${tok}` + excessiveDebugOutput ? `, obj=${JSON.stringify(serialized)}` : ``);
const req = await createCallRequest(tok, serialized, propertyDepsResources, provider, version);

15
sdk/nodejs/runtime/resource.ts
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -205,7 +205,7 @@ export function readResource(res: Resource, t: string, name: string, props: Inpu
const preallocError = new Error();
debuggablePromise(resopAsync.then(async (resop) => {
const resolvedID = await serializeProperty(label, id, new Set());
const resolvedID = await serializeProperty(label, id, new Set(), { keepOutputValues: false });
log.debug(`ReadResource RPC prepared: id=${resolvedID}, t=${t}, name=${name}` +
(excessiveDebugOutput ? `, obj=${JSON.stringify(resop.serializedProps)}` : ``));
@ -510,7 +510,11 @@ async function prepareResource(label: string, res: Resource, custom: boolean, re
// Serialize out all our props to their final values. In doing so, we'll also collect all
// the Resources pointed to by any Dependency objects we encounter, adding them to 'propertyDependencies'.
const [serializedProps, propertyToDirectDependencies] = await serializeResourceProperties(label, props);
const [serializedProps, propertyToDirectDependencies] = await serializeResourceProperties(label, props, {
// To initially scope the use of this new feature, we only keep output values when
// remote is true (for multi-lang components).
keepOutputValues: remote,
});
// Wait for the parent to complete.
// If no parent was provided, parent to the root resource.
@ -596,7 +600,8 @@ function addAll<T>(to: Set<T>, from: Set<T>) {
}
}
async function getAllTransitivelyReferencedResourceURNs(resources: Set<Resource>): Promise<Set<string>> {
/** @internal */
export async function getAllTransitivelyReferencedResourceURNs(resources: Set<Resource>): Promise<Set<string>> {
// Go through 'resources', but transitively walk through **Component** resources, collecting any
// of their child resources. This way, a Component acts as an aggregation really of all the
// reachable resources it parents. This walking will stop when it hits custom resources.
@ -720,7 +725,7 @@ async function resolveOutputs(res: Resource, t: string, name: string,
// input prop the engine didn't give us a final value for. Just use the value passed into the resource
// after round-tripping it through serialization. We do the round-tripping primarily s.t. we ensure that
// Output values are handled properly w.r.t. unknowns.
const inputProp = await serializeProperty(label, props[key], new Set());
const inputProp = await serializeProperty(label, props[key], new Set(), { keepOutputValues: false });
if (inputProp === undefined) {
continue;
}

119
sdk/nodejs/runtime/rpc.ts
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -18,7 +18,9 @@ import * as log from "../log";
import { getAllResources, Input, Inputs, isUnknown, Output, unknown } from "../output";
import { ComponentResource, CustomResource, ProviderResource, Resource, URN } from "../resource";
import { debuggablePromise, errorString, promiseDebugString } from "./debuggable";
import { excessiveDebugOutput, isDryRun, monitorSupportsResourceReferences, monitorSupportsSecrets } from "./settings";
import { excessiveDebugOutput, isDryRun, monitorSupportsOutputValues, monitorSupportsResourceReferences,
monitorSupportsSecrets } from "./settings";
import { getAllTransitivelyReferencedResourceURNs } from "./resource";
import * as semver from "semver";
@ -106,6 +108,17 @@ export function transferProperties(onto: Resource, label: string, props: Inputs)
return resolvers;
}
/**
* Controls the serialization of RPC structures.
*/
export interface SerializationOptions {
/**
* true if we are keeping output values.
* If the monitor does not support output values, they will not be kept, even when this is set to true.
*/
keepOutputValues?: boolean;
}
/**
* serializeFilteredProperties walks the props object passed in, awaiting all interior promises for
* properties with keys that match the provided filter, creating a reasonable POJO object that can
@ -115,6 +128,7 @@ async function serializeFilteredProperties(
label: string,
props: Inputs,
acceptKey: (k: string) => boolean,
opts?: SerializationOptions,
): Promise<[Record<string, any>, Map<string, Set<Resource>>]> {
const propertyToDependentResources = new Map<string, Set<Resource>>();
@ -124,7 +138,7 @@ async function serializeFilteredProperties(
if (acceptKey(k)) {
// We treat properties with undefined values as if they do not exist.
const dependentResources = new Set<Resource>();
const v = await serializeProperty(`${label}.${k}`, props[k], dependentResources);
const v = await serializeProperty(`${label}.${k}`, props[k], dependentResources, opts);
if (v !== undefined) {
result[k] = v;
propertyToDependentResources.set(k, dependentResources);
@ -139,22 +153,22 @@ async function serializeFilteredProperties(
* serializeResourceProperties walks the props object passed in, awaiting all interior promises besides those for `id`
* and `urn`, creating a reasonable POJO object that can be remoted over to registerResource.
*/
export async function serializeResourceProperties(label: string, props: Inputs) {
return serializeFilteredProperties(label, props, key => key !== "id" && key !== "urn");
export async function serializeResourceProperties(label: string, props: Inputs, opts?: SerializationOptions) {
return serializeFilteredProperties(label, props, key => key !== "id" && key !== "urn", opts);
}
/**
* serializeProperties walks the props object passed in, awaiting all interior promises, creating a reasonable
* POJO object that can be remoted over to registerResource.
*/
export async function serializeProperties(label: string, props: Inputs) {
const [result] = await serializeFilteredProperties(label, props, _ => true);
export async function serializeProperties(label: string, props: Inputs, opts?: SerializationOptions) {
const [result] = await serializeFilteredProperties(label, props, _ => true, opts);
return result;
}
/** @internal */
export async function serializePropertiesReturnDeps(label: string, props: Inputs) {
return serializeFilteredProperties(label, props, _ => true);
export async function serializePropertiesReturnDeps(label: string, props: Inputs, opts?: SerializationOptions) {
return serializeFilteredProperties(label, props, _ => true, opts);
}
/**
@ -254,31 +268,39 @@ export function resolveProperties(
*/
export const unknownValue = "04da6b54-80e4-46f7-96ec-b56ff0331ba9";
/**
* specialSigKey is sometimes used to encode type identity inside of a map. See pkg/resource/properties.go.
* specialSigKey is sometimes used to encode type identity inside of a map. See sdk/go/common/resource/properties.go.
*/
export const specialSigKey = "4dabf18193072939515e22adb298388d";
/**
* specialAssetSig is a randomly assigned hash used to identify assets in maps. See pkg/resource/asset.go.
* specialAssetSig is a randomly assigned hash used to identify assets in maps. See sdk/go/common/resource/asset.go.
*/
export const specialAssetSig = "c44067f5952c0a294b673a41bacd8c17";
/**
* specialArchiveSig is a randomly assigned hash used to identify archives in maps. See pkg/resource/asset.go.
* specialArchiveSig is a randomly assigned hash used to identify archives in maps. See sdk/go/common/resource/asset.go.
*/
export const specialArchiveSig = "0def7320c3a5731c473e5ecbe6d01bc7";
/**
* specialSecretSig is a randomly assigned hash used to identify secrets in maps. See pkg/resource/properties.go.
* specialSecretSig is a randomly assigned hash used to identify secrets in maps.
* See sdk/go/common/resource/properties.go.
*/
export const specialSecretSig = "1b47061264138c4ac30d75fd1eb44270";
/**
* specialResourceSig is a randomly assigned hash used to identify resources in maps. See pkg/resource/properties.go.
* specialResourceSig is a randomly assigned hash used to identify resources in maps.
* See sdk/go/common/resource/properties.go.
*/
export const specialResourceSig = "5cf8f73096256a8f31e491e813e4eb8e";
/**
* specialOutputValueSig is a randomly assigned hash used to identify outputs in maps.
* See sdk/go/common/resource/properties.go.
*/
export const specialOutputValueSig = "d0e6a833031e9bbcd3f4e8bde6ca49a4";
/**
* serializeProperty serializes properties deeply. This understands how to wait on any unresolved promises, as
* appropriate, in addition to translating certain "special" values so that they are ready to go on the wire.
*/
export async function serializeProperty(ctx: string, prop: Input<any>, dependentResources: Set<Resource>): Promise<any> {
export async function serializeProperty(
ctx: string, prop: Input<any>, dependentResources: Set<Resource>, opts?: SerializationOptions): Promise<any> {
// IMPORTANT:
// IMPORTANT: Keep this in sync with serializePropertiesSync in invoke.ts
// IMPORTANT:
@ -302,7 +324,7 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
[specialSigKey]: asset.Asset.isInstance(prop) ? specialAssetSig : specialArchiveSig,
};
return await serializeAllKeys(prop, obj);
return await serializeAllKeys(prop, obj, { keepOutputValues: false });
}
if (prop instanceof Promise) {
@ -313,7 +335,7 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
const subctx = `Promise<${ctx}>`;
return serializeProperty(subctx,
await debuggablePromise(prop, `serializeProperty.await(${subctx})`), dependentResources);
await debuggablePromise(prop, `serializeProperty.await(${subctx})`), dependentResources, opts);
}
if (Output.isInstance(prop)) {
@ -340,7 +362,44 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
// which will wrap undefined, if it were to be resolved (since `Output` has no member named .isSecret).
// so we must compare to the literal true instead of just doing await prop.isSecret.
const isSecret = await prop.isSecret === true;
const value = await serializeProperty(`${ctx}.id`, prop.promise(), dependentResources);
const promiseDeps = new Set<Resource>();
const value = await serializeProperty(`${ctx}.id`, prop.promise(), promiseDeps, {
keepOutputValues: false,
});
for (const resource of promiseDeps) {
propResources.add(resource);
dependentResources.add(resource);
}
if (opts?.keepOutputValues && await monitorSupportsOutputValues()) {
const urnDeps = new Set<Resource>();
for (const resource of propResources) {
await serializeProperty(`${ctx} dependency`, resource.urn, urnDeps, {
keepOutputValues: false,
});
}
for (const resource of urnDeps) {
propResources.add(resource);
dependentResources.add(resource);
}
const dependencies = await getAllTransitivelyReferencedResourceURNs(propResources);
const obj: any = {
[specialSigKey]: specialOutputValueSig,
};
if (isKnown) {
// coerce 'undefined' to 'null' as required by the protobuf system.
obj["value"] = value === undefined ? null : value;
}
if (isSecret) {
obj["secret"] = isSecret;
}
if (dependencies.size > 0) {
obj["dependencies"] = Array.from(dependencies);
}
return obj;
}
if (!isKnown) {
return unknownValue;
@ -365,11 +424,15 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
}
dependentResources.add(prop);
const id = await serializeProperty(`${ctx}.id`, prop.id, dependentResources);
const id = await serializeProperty(`${ctx}.id`, prop.id, dependentResources, {
keepOutputValues: false,
});
if (await monitorSupportsResourceReferences()) {
// If we are keeping resources, emit a stronly typed wrapper over the URN
const urn = await serializeProperty(`${ctx}.urn`, prop.urn, dependentResources);
const urn = await serializeProperty(`${ctx}.urn`, prop.urn, dependentResources, {
keepOutputValues: false,
});
return {
[specialSigKey]: specialResourceSig,
urn: urn,
@ -401,14 +464,18 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
if (await monitorSupportsResourceReferences()) {
// If we are keeping resources, emit a strongly typed wrapper over the URN
const urn = await serializeProperty(`${ctx}.urn`, prop.urn, dependentResources);
const urn = await serializeProperty(`${ctx}.urn`, prop.urn, dependentResources, {
keepOutputValues: false,
});
return {
[specialSigKey]: specialResourceSig,
urn: urn,
};
}
// Else, return the urn for backward compatibility.
return serializeProperty(`${ctx}.urn`, prop.urn, dependentResources);
return serializeProperty(`${ctx}.urn`, prop.urn, dependentResources, {
keepOutputValues: false,
});
}
if (prop instanceof Array) {
@ -418,22 +485,22 @@ export async function serializeProperty(ctx: string, prop: Input<any>, dependent
log.debug(`Serialize property [${ctx}]: array[${i}] element`);
}
// When serializing arrays, we serialize any undefined values as `null`. This matches JSON semantics.
const elem = await serializeProperty(`${ctx}[${i}]`, prop[i], dependentResources);
const elem = await serializeProperty(`${ctx}[${i}]`, prop[i], dependentResources, opts);
result.push(elem === undefined ? null : elem);
}
return result;
}
return await serializeAllKeys(prop, {});
return await serializeAllKeys(prop, {}, opts);
async function serializeAllKeys(innerProp: any, obj: any) {
async function serializeAllKeys(innerProp: any, obj: any, innerOpts?: SerializationOptions) {
for (const k of Object.keys(innerProp)) {
if (excessiveDebugOutput) {
log.debug(`Serialize property [${ctx}]: object.${k}`);
}
// When serializing an object, we omit any keys with undefined values. This matches JSON semantics.
const v = await serializeProperty(`${ctx}.${k}`, innerProp[k], dependentResources);
const v = await serializeProperty(`${ctx}.${k}`, innerProp[k], dependentResources, innerOpts);
if (v !== undefined) {
obj[k] = v;
}

11
sdk/nodejs/runtime/settings.ts
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -557,6 +557,15 @@ export async function monitorSupportsResourceReferences(): Promise<boolean> {
return monitorSupportsFeature("resourceReferences");
}
/**
* monitorSupportsOutputValues returns a promise that when resolved tells you if the resource monitor we are
* connected to is able to support output values across its RPC interface. When it does, we marshal outputs
* in a special way.
*/
export async function monitorSupportsOutputValues(): Promise<boolean> {
return monitorSupportsFeature("outputValues");
}
// sxsRandomIdentifier is a module level global that is transfered to process.env.
// the goal is to detect side by side (sxs) pulumi/pulumi situations for inline programs
// and fail fast. See https://github.com/pulumi/pulumi/issues/7333 for details.

54
sdk/nodejs/tests/runtime/props.spec.ts
View file

@ -1,4 +1,4 @@
// Copyright 2016-2018, Pulumi Corporation.
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -13,7 +13,8 @@
// limitations under the License.
import * as assert from "assert";
import { ComponentResource, CustomResource, Inputs, Resource, ResourceOptions, runtime, secret } from "../../index";
import { ComponentResource, CustomResource, DependencyResource, Inputs, Output, Resource, ResourceOptions, runtime,
secret } from "../../index";
import { asyncTest } from "../util";
const gstruct = require("google-protobuf/google/protobuf/struct_pb.js");
@ -131,6 +132,55 @@ describe("runtime", () => {
});
describe("transferProperties", () => {
describe("output values", () => {
function* generateTests() {
const testValues = [
{ value: undefined, expected: null },
{ value: null, expected: null },
{ value: 0, expected: 0 },
{ value: 1, expected: 1 },
{ value: "", expected: "" },
{ value: "hi", expected: "hi" },
{ value: {}, expected: {} },
{ value: [], expected: [] },
];
for (const tv of testValues) {
for (const deps of [[], ["fakeURN1", "fakeURN2"]]) {
for (const isKnown of [true, false]) {
for (const isSecret of [true, false])
{
const resources = deps.map(dep => new DependencyResource(dep));
yield {
name: `Output(${JSON.stringify(deps)}, ${JSON.stringify(tv.value)}, ` +
`isKnown=${isKnown}, isSecret=${isSecret})`,
input: new Output(resources, Promise.resolve(tv.value), Promise.resolve(isKnown),
Promise.resolve(isSecret), Promise.resolve([])),
expected: {
[runtime.specialSigKey]: runtime.specialOutputValueSig,
...isKnown && { value: tv.expected },
...isSecret && { secret: isSecret },
...(deps.length > 0) && { dependencies: deps },
},
};
}
}
}
}
}
for (const test of generateTests()) {
it(`marshals ${test.name} correctly`, asyncTest(async () => {
runtime._setTestModeEnabled(true);
runtime._setFeatureSupport("outputValues", true);
const inputs = { value: test.input };
const expected = { value: test.expected };
const actual = await runtime.serializeProperties("test", inputs, { keepOutputValues: true });
assert.deepStrictEqual(actual, expected);
}));
}
});
it("marshals basic properties correctly", asyncTest(async () => {
const inputs: TestInputs = {
"aNum": 42,