This update includes several changes to core `@pulumi/pulumi` constructs that will not play nicely in side-by-side applications that pull in prior versions of this package. As such, we are rev'ing the minor version of the package from 0.16 to 0.17. Recent version of `pulumi` will now detect, and warn, if different versions of `@pulumi/pulumi` are loaded into the same application. If you encounter this warning, it is recommended you move to versions of the `@pulumi/...` packages that are compatible. i.e. keep everything on 0.16.x until you are ready to move everything to 0.17.x. ### Improvements - `Output<T>` now 'lifts' property members from the value it wraps, simplifying common coding patterns. Note: this wrapping only happens for POJO values, not `Output<Resource>`s. - Depending on a **Component** Resource will now depend on all other Resources parented by that Resource. This will help out the programming model for Component Resources as your consumers can just depend on a Component and have that automatically depend on all the child Resources created by that Component. Note: this does not apply to a **Custom** resource. Depending on a CustomResource will still only wait on that single resource being created, not any other Resources that consider that CustomResource to be a parent.
492 lines
23 KiB
TypeScript
492 lines
23 KiB
TypeScript
// Copyright 2016-2018, Pulumi Corporation.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
import * as grpc from "grpc";
|
|
import * as log from "../log";
|
|
import { Input, Inputs, Output } from "../output";
|
|
import { ComponentResource, CustomResource, CustomResourceOptions, ID, Resource, ResourceOptions, URN } from "../resource";
|
|
import { debuggablePromise } from "./debuggable";
|
|
|
|
import {
|
|
deserializeProperties,
|
|
deserializeProperty,
|
|
OutputResolvers,
|
|
resolveProperties,
|
|
serializeProperties,
|
|
serializeProperty,
|
|
serializeResourceProperties,
|
|
transferProperties,
|
|
unknownValue,
|
|
} from "./rpc";
|
|
import { excessiveDebugOutput, getMonitor, getRootResource, rpcKeepAlive, serialize } from "./settings";
|
|
|
|
const gstruct = require("google-protobuf/google/protobuf/struct_pb.js");
|
|
const resproto = require("../proto/resource_pb.js");
|
|
|
|
interface ResourceResolverOperation {
|
|
// A resolver for a resource's URN.
|
|
resolveURN: (urn: URN) => void;
|
|
// A resolver for a resource's ID (for custom resources only).
|
|
resolveID: ((v: ID, performApply: boolean) => void) | undefined;
|
|
// A collection of resolvers for a resource's properties.
|
|
resolvers: OutputResolvers;
|
|
// A parent URN, fully resolved, if any.
|
|
parentURN: URN | undefined;
|
|
// A provider reference, fully resolved, if any.
|
|
providerRef: string | undefined;
|
|
// All serialized properties, fully awaited, serialized, and ready to go.
|
|
serializedProps: Record<string, any>;
|
|
// A set of URNs that this resource is directly dependent upon. These will all be URNs of
|
|
// custom resources, not component resources.
|
|
allDirectDependencyURNs: Set<URN>;
|
|
// Set of URNs that this resource is directly dependent upon, keyed by the property that causes
|
|
// the dependency. All urns in this map must exist in [allDirectDependencyURNs]. These will
|
|
// all be URNs of custom resources, not component resources.
|
|
propertyToDirectDependencyURNs: Map<string, Set<URN>>;
|
|
}
|
|
|
|
/**
|
|
* Reads an existing custom resource's state from the resource monitor. Note that resources read in this way
|
|
* will not be part of the resulting stack's state, as they are presumed to belong to another.
|
|
*/
|
|
export function readResource(res: Resource, t: string, name: string, props: Inputs, opts: ResourceOptions): void {
|
|
const id: Input<ID> | undefined = opts.id;
|
|
if (!id) {
|
|
throw new Error("Cannot read resource whose options are lacking an ID value");
|
|
}
|
|
|
|
const label = `resource:${name}[${t}]#...`;
|
|
log.debug(`Reading resource: id=${Output.isInstance(id) ? "Output<T>" : id}, t=${t}, name=${name}`);
|
|
|
|
const monitor: any = getMonitor();
|
|
const resopAsync = prepareResource(label, res, true, props, opts);
|
|
|
|
const preallocError = new Error();
|
|
debuggablePromise(resopAsync.then(async (resop) => {
|
|
const resolvedID = await serializeProperty(label, id, new Set());
|
|
log.debug(`ReadResource RPC prepared: id=${resolvedID}, t=${t}, name=${name}` +
|
|
(excessiveDebugOutput ? `, obj=${JSON.stringify(resop.serializedProps)}` : ``));
|
|
|
|
// Create a resource request and do the RPC.
|
|
const req = new resproto.ReadResourceRequest();
|
|
req.setType(t);
|
|
req.setName(name);
|
|
req.setId(resolvedID);
|
|
req.setParent(resop.parentURN);
|
|
req.setProvider(resop.providerRef);
|
|
req.setProperties(gstruct.Struct.fromJavaScript(resop.serializedProps));
|
|
req.setDependenciesList(Array.from(resop.allDirectDependencyURNs));
|
|
|
|
// Now run the operation, serializing the invocation if necessary.
|
|
const opLabel = `monitor.readResource(${label})`;
|
|
runAsyncResourceOp(opLabel, async () => {
|
|
const resp: any = await debuggablePromise(new Promise((resolve, reject) =>
|
|
monitor.readResource(req, (err: Error, innerResponse: any) => {
|
|
log.debug(`ReadResource RPC finished: ${label}; err: ${err}, resp: ${innerResponse}`);
|
|
if (err) {
|
|
preallocError.message =
|
|
`failed to read resource #${resolvedID} '${name}' [${t}]: ${err.message}`;
|
|
reject(preallocError);
|
|
}
|
|
else {
|
|
resolve(innerResponse);
|
|
}
|
|
})), opLabel);
|
|
|
|
// Now resolve everything: the URN, the ID (supplied as input), and the output properties.
|
|
resop.resolveURN(resp.getUrn());
|
|
resop.resolveID!(resolvedID, resolvedID !== undefined);
|
|
await resolveOutputs(res, t, name, props, resp.getProperties(), resop.resolvers);
|
|
});
|
|
}), label);
|
|
}
|
|
|
|
/**
|
|
* registerResource registers a new resource object with a given type t and name. It returns the auto-generated
|
|
* URN and the ID that will resolve after the deployment has completed. All properties will be initialized to property
|
|
* objects that the registration operation will resolve at the right time (or remain unresolved for deployments).
|
|
*/
|
|
export function registerResource(res: Resource, t: string, name: string, custom: boolean,
|
|
props: Inputs, opts: ResourceOptions): void {
|
|
const label = `resource:${name}[${t}]`;
|
|
log.debug(`Registering resource: t=${t}, name=${name}, custom=${custom}`);
|
|
|
|
const monitor: any = getMonitor();
|
|
const resopAsync = prepareResource(label, res, custom, props, opts);
|
|
|
|
// In order to present a useful stack trace if an error does occur, we preallocate potential
|
|
// errors here. V8 captures a stack trace at the moment an Error is created and this stack
|
|
// trace will lead directly to user code. Throwing in `runAsyncResourceOp` results in an Error
|
|
// with a non-useful stack trace.
|
|
const preallocError = new Error();
|
|
debuggablePromise(resopAsync.then(async (resop) => {
|
|
log.debug(`RegisterResource RPC prepared: t=${t}, name=${name}` +
|
|
(excessiveDebugOutput ? `, obj=${JSON.stringify(resop.serializedProps)}` : ``));
|
|
|
|
const req = new resproto.RegisterResourceRequest();
|
|
req.setType(t);
|
|
req.setName(name);
|
|
req.setParent(resop.parentURN);
|
|
req.setCustom(custom);
|
|
req.setObject(gstruct.Struct.fromJavaScript(resop.serializedProps));
|
|
req.setProtect(opts.protect);
|
|
req.setProvider(resop.providerRef);
|
|
req.setDependenciesList(Array.from(resop.allDirectDependencyURNs));
|
|
req.setDeletebeforereplace((<any>opts).deleteBeforeReplace || false);
|
|
|
|
const propertyDependencies = req.getPropertydependenciesMap();
|
|
for (const [key, resourceURNs] of resop.propertyToDirectDependencyURNs) {
|
|
const deps = new resproto.RegisterResourceRequest.PropertyDependencies();
|
|
deps.setUrnsList(Array.from(resourceURNs));
|
|
propertyDependencies.set(key, deps);
|
|
}
|
|
|
|
// Now run the operation, serializing the invocation if necessary.
|
|
const opLabel = `monitor.registerResource(${label})`;
|
|
runAsyncResourceOp(opLabel, async () => {
|
|
const resp: any = await debuggablePromise(new Promise((resolve, reject) =>
|
|
monitor.registerResource(req, (err: grpc.ServiceError, innerResponse: any) => {
|
|
log.debug(`RegisterResource RPC finished: ${label}; err: ${err}, resp: ${innerResponse}`);
|
|
if (err) {
|
|
// If the monitor is unavailable, it is in the process of shutting down or has already
|
|
// shut down. Don't emit an error and don't do any more RPCs, just exit.
|
|
if (err.code === grpc.status.UNAVAILABLE) {
|
|
log.debug("Resource monitor is terminating");
|
|
process.exit(0);
|
|
}
|
|
|
|
// Node lets us hack the message as long as we do it before accessing the `stack` property.
|
|
preallocError.message = `failed to register new resource ${name} [${t}]: ${err.message}`;
|
|
reject(preallocError);
|
|
}
|
|
else {
|
|
resolve(innerResponse);
|
|
}
|
|
})), opLabel);
|
|
|
|
resop.resolveURN(resp.getUrn());
|
|
|
|
// Note: 'id || undefined' is intentional. We intentionally collapse falsy values to
|
|
// undefined so that later parts of our system don't have to deal with values like 'null'.
|
|
if (resop.resolveID) {
|
|
const id = resp.getId() || undefined;
|
|
resop.resolveID(id, id !== undefined);
|
|
}
|
|
|
|
// Now resolve the output properties.
|
|
await resolveOutputs(res, t, name, props, resp.getObject(), resop.resolvers);
|
|
});
|
|
}), label);
|
|
}
|
|
|
|
/**
|
|
* Prepares for an RPC that will manufacture a resource, and hence deals with input and output
|
|
* properties.
|
|
*/
|
|
async function prepareResource(label: string, res: Resource, custom: boolean,
|
|
props: Inputs, opts: ResourceOptions): Promise<ResourceResolverOperation> {
|
|
|
|
// Simply initialize the URN property and get prepared to resolve it later on.
|
|
// Note: a resource urn will always get a value, and thus the output property
|
|
// for it can always run .apply calls.
|
|
let resolveURN: (urn: URN) => void;
|
|
(res as any).urn = new Output(
|
|
res,
|
|
debuggablePromise(
|
|
new Promise<URN>(resolve => resolveURN = resolve),
|
|
`resolveURN(${label})`),
|
|
/*performApply:*/ Promise.resolve(true));
|
|
|
|
// If a custom resource, make room for the ID property.
|
|
let resolveID: ((v: any, performApply: boolean) => void) | undefined;
|
|
if (custom) {
|
|
let resolveValue: (v: ID) => void;
|
|
let resolvePerformApply: (v: boolean) => void;
|
|
(res as any).id = new Output(
|
|
res,
|
|
debuggablePromise(new Promise<ID>(resolve => resolveValue = resolve), `resolveID(${label})`),
|
|
debuggablePromise(new Promise<boolean>(
|
|
resolve => resolvePerformApply = resolve), `resolveIDPerformApply(${label})`));
|
|
|
|
resolveID = (v, performApply) => {
|
|
resolveValue(v);
|
|
resolvePerformApply(performApply);
|
|
};
|
|
}
|
|
|
|
// Now "transfer" all input properties into unresolved Promises on res. This way,
|
|
// this resource will look like it has all its output properties to anyone it is
|
|
// passed to. However, those promises won't actually resolve until the registerResource
|
|
// RPC returns
|
|
const resolvers = transferProperties(res, label, props);
|
|
|
|
/** IMPORTANT! We should never await prior to this line, otherwise the Resource will be partly uninitialized. */
|
|
|
|
// Before we can proceed, all our dependencies must be finished.
|
|
const explicitDirectDependencies = new Set(await gatherExplicitDependencies(opts.dependsOn));
|
|
|
|
// Serialize out all our props to their final values. In doing so, we'll also collect all
|
|
// the Resources pointed to by any Dependency objects we encounter, adding them to 'propertyDependencies'.
|
|
const [serializedProps, propertyToDirectDependencies] = await serializeResourceProperties(label, props);
|
|
|
|
// Wait for the parent to complete.
|
|
// If no parent was provided, parent to the root resource.
|
|
const parentURN = opts.parent
|
|
? await opts.parent.urn.promise()
|
|
: await getRootResource();
|
|
|
|
let providerRef: string | undefined;
|
|
if (custom && (<CustomResourceOptions>opts).provider) {
|
|
const provider = (<CustomResourceOptions>opts).provider!;
|
|
const providerURN = await provider.urn.promise();
|
|
const providerID = await provider.id.promise() || unknownValue;
|
|
providerRef = `${providerURN}::${providerID}`;
|
|
}
|
|
|
|
// Collect the URNs for explicit/implicit dependencies for the engine so that it can understand
|
|
// the dependency graph and optimize operations accordingly.
|
|
|
|
// The list of all dependencies (implicit or explicit).
|
|
const allDirectDependencies = new Set<Resource>(explicitDirectDependencies);
|
|
|
|
const allDirectDependencyURNs = await getAllTransitivelyReferencedCustomResourceURNs(explicitDirectDependencies);
|
|
const propertyToDirectDependencyURNs = new Map<string, Set<URN>>();
|
|
|
|
for (const [propertyName, directDependencies] of propertyToDirectDependencies) {
|
|
addAll(allDirectDependencies, directDependencies);
|
|
|
|
const urns = await getAllTransitivelyReferencedCustomResourceURNs(directDependencies);
|
|
addAll(allDirectDependencyURNs, urns);
|
|
propertyToDirectDependencyURNs.set(propertyName, urns);
|
|
}
|
|
|
|
return {
|
|
resolveURN: resolveURN!,
|
|
resolveID: resolveID,
|
|
resolvers: resolvers,
|
|
serializedProps: serializedProps,
|
|
parentURN: parentURN,
|
|
providerRef: providerRef,
|
|
allDirectDependencyURNs: allDirectDependencyURNs,
|
|
propertyToDirectDependencyURNs: propertyToDirectDependencyURNs,
|
|
};
|
|
}
|
|
|
|
function addAll<T>(to: Set<T>, from: Set<T>) {
|
|
for (const val of from) {
|
|
to.add(val);
|
|
}
|
|
}
|
|
|
|
async function getAllTransitivelyReferencedCustomResourceURNs(resources: Set<Resource>) {
|
|
// Go through 'resources', but transitively walk through **Component** resources, collecting any
|
|
// of their child resources. This way, a Component acts as an aggregation really of all the
|
|
// reachable custom resources it parents. This walking will transitively walk through other
|
|
// child ComponentResources, but will stop when it hits custom resources. in other words, if we
|
|
// had:
|
|
//
|
|
// Comp1
|
|
// / \
|
|
// Cust1 Comp2
|
|
// / \
|
|
// Cust2 Cust3
|
|
// /
|
|
// Cust4
|
|
//
|
|
// Then the transitively reachable custom resources of Comp1 will be [Cust1, Cust2, Cust3]. It
|
|
// will *not* include `Cust4`.
|
|
|
|
// To do this, first we just get the transitively reachable set of resources (not diving
|
|
// into custom resources). In the above picture, if we start with 'Comp1', this will be
|
|
// [Comp1, Cust1, Comp2, Cust2, Cust3]
|
|
const transitivelyReachableResources = getTransitivelyReferencedChildResourcesOfComponentResources(resources);
|
|
|
|
const transitivelyReacableCustomResources = [...transitivelyReachableResources].filter(r => CustomResource.isInstance(r));
|
|
const promises = transitivelyReacableCustomResources.map(r => r.urn.promise());
|
|
const urns = await Promise.all(promises);
|
|
return new Set<string>(urns);
|
|
}
|
|
|
|
/**
|
|
* Recursively walk the resources passed in, returning them and all resources reachable from
|
|
* [Resource.__childResources] through any **Component** resources we encounter.
|
|
*/
|
|
function getTransitivelyReferencedChildResourcesOfComponentResources(resources: Set<Resource>) {
|
|
// Recursively walk the dependent resources through their children, adding them to the result set.
|
|
const result = new Set<Resource>();
|
|
addTransitivelyReferencedChildResourcesOfComponentResources(resources, result);
|
|
return result;
|
|
}
|
|
|
|
function addTransitivelyReferencedChildResourcesOfComponentResources(resources: Set<Resource> | undefined, result: Set<Resource>) {
|
|
if (resources) {
|
|
for (const resource of resources) {
|
|
if (!result.has(resource)) {
|
|
result.add(resource);
|
|
|
|
if (ComponentResource.isInstance(resource)) {
|
|
addTransitivelyReferencedChildResourcesOfComponentResources(resource.__childResources, result);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Gathers explicit dependent Resources from a list of Resources (possibly Promises and/or Outputs).
|
|
*/
|
|
async function gatherExplicitDependencies(
|
|
dependsOn: Input<Input<Resource>[]> | Input<Resource> | undefined): Promise<Resource[]> {
|
|
|
|
if (dependsOn) {
|
|
if (Array.isArray(dependsOn)) {
|
|
const dos: Resource[] = [];
|
|
for (const d of dependsOn) {
|
|
dos.push(...(await gatherExplicitDependencies(d)));
|
|
}
|
|
return dos;
|
|
} else if (dependsOn instanceof Promise) {
|
|
return gatherExplicitDependencies(await dependsOn);
|
|
} else if (Output.isInstance(dependsOn)) {
|
|
// Recursively gather dependencies, await the promise, and append the output's dependencies.
|
|
const dos = (dependsOn as Output<Input<Resource>[] | Input<Resource>>).apply(gatherExplicitDependencies);
|
|
const urns = await dos.promise();
|
|
const implicits = await gatherExplicitDependencies([...dos.resources()]);
|
|
return urns.concat(implicits);
|
|
} else {
|
|
return [dependsOn as Resource];
|
|
}
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
/**
|
|
* Finishes a resource creation RPC operation by resolving its outputs to the resulting RPC payload.
|
|
*/
|
|
async function resolveOutputs(res: Resource, t: string, name: string,
|
|
props: Inputs, outputs: any, resolvers: OutputResolvers): Promise<void> {
|
|
// Produce a combined set of property states, starting with inputs and then applying
|
|
// outputs. If the same property exists in the inputs and outputs states, the output wins.
|
|
const allProps: Record<string, any> = {};
|
|
if (outputs) {
|
|
Object.assign(allProps, deserializeProperties(outputs));
|
|
}
|
|
|
|
const label = `resource:${name}[${t}]#...`;
|
|
for (const key of Object.keys(props)) {
|
|
if (!allProps.hasOwnProperty(key)) {
|
|
// input prop the engine didn't give us a final value for. Just use the value passed into the resource
|
|
// after round-tripping it through serialization. We do the round-tripping primarily s.t. we ensure that
|
|
// Output values are handled properly w.r.t. unknowns.
|
|
const inputProp = await serializeProperty(label, props[key], new Set());
|
|
if (inputProp === undefined) {
|
|
continue;
|
|
}
|
|
allProps[key] = deserializeProperty(inputProp);
|
|
}
|
|
}
|
|
|
|
resolveProperties(res, resolvers, t, name, allProps);
|
|
}
|
|
|
|
/**
|
|
* registerResourceOutputs completes the resource registration, attaching an optional set of computed outputs.
|
|
*/
|
|
export function registerResourceOutputs(res: Resource, outputs: Inputs | Promise<Inputs> | Output<Inputs>) {
|
|
// Now run the operation. Note that we explicitly do not serialize output registration with
|
|
// respect to other resource operations, as outputs may depend on properties of other resources
|
|
// that will not resolve until later turns. This would create a circular promise chain that can
|
|
// never resolve.
|
|
const opLabel = `monitor.registerResourceOutputs(...)`;
|
|
runAsyncResourceOp(opLabel, async () => {
|
|
// The registration could very well still be taking place, so we will need to wait for its URN.
|
|
// Additionally, the output properties might have come from other resources, so we must await those too.
|
|
const urn = await res.urn.promise();
|
|
const resolved = await serializeProperties(opLabel, { outputs });
|
|
const outputsObj = gstruct.Struct.fromJavaScript(resolved.outputs);
|
|
log.debug(`RegisterResourceOutputs RPC prepared: urn=${urn}` +
|
|
(excessiveDebugOutput ? `, outputs=${JSON.stringify(outputsObj)}` : ``));
|
|
|
|
// Fetch the monitor and make an RPC request.
|
|
const monitor: any = getMonitor();
|
|
|
|
const req = new resproto.RegisterResourceOutputsRequest();
|
|
req.setUrn(urn);
|
|
req.setOutputs(outputsObj);
|
|
|
|
const label = `monitor.registerResourceOutputs(${urn}, ...)`;
|
|
await debuggablePromise(new Promise((resolve, reject) =>
|
|
monitor.registerResourceOutputs(req, (err: grpc.ServiceError, innerResponse: any) => {
|
|
log.debug(`RegisterResourceOutputs RPC finished: urn=${urn}; `+
|
|
`err: ${err}, resp: ${innerResponse}`);
|
|
if (err) {
|
|
// If the monitor is unavailable, it is in the process of shutting down or has already
|
|
// shut down. Don't emit an error and don't do any more RPCs, just exit.
|
|
if (err.code === grpc.status.UNAVAILABLE) {
|
|
log.debug("Resource monitor is terminating");
|
|
process.exit(0);
|
|
}
|
|
|
|
log.error(`Failed to end new resource registration '${urn}': ${err.stack}`);
|
|
reject(err);
|
|
}
|
|
else {
|
|
resolve();
|
|
}
|
|
})), label);
|
|
}, false);
|
|
}
|
|
|
|
/**
|
|
* resourceChain is used to serialize all resource requests. If we don't do this, all resource operations will be
|
|
* entirely asynchronous, meaning the dataflow graph that results will determine ordering of operations. This
|
|
* causes problems with some resource providers, so for now we will serialize all of them. The issue
|
|
* pulumi/pulumi#335 tracks coming up with a long-term solution here.
|
|
*/
|
|
let resourceChain: Promise<void> = Promise.resolve();
|
|
let resourceChainLabel: string | undefined = undefined;
|
|
|
|
// runAsyncResourceOp runs an asynchronous resource operation, possibly serializing it as necessary.
|
|
function runAsyncResourceOp(label: string, callback: () => Promise<void>, serial?: boolean): void {
|
|
// Serialize the invocation if necessary.
|
|
if (serial === undefined) {
|
|
serial = serialize();
|
|
}
|
|
const resourceOp: Promise<void> = debuggablePromise(resourceChain.then(async () => {
|
|
if (serial) {
|
|
resourceChainLabel = label;
|
|
log.debug(`Resource RPC serialization requested: ${label} is current`);
|
|
}
|
|
return callback();
|
|
}), label + "-initial");
|
|
|
|
// Ensure the process won't exit until this RPC call finishes and resolve it when appropriate.
|
|
const done: () => void = rpcKeepAlive();
|
|
const finalOp: Promise<void> = debuggablePromise(
|
|
resourceOp.then(() => { done(); }, () => { done(); }),
|
|
label + "-final");
|
|
|
|
// Set up another promise that propagates the error, if any, so that it triggers unhandled rejection logic.
|
|
resourceOp.catch((err) => Promise.reject(err));
|
|
|
|
// If serialization is requested, wait for the prior resource operation to finish before we proceed, serializing
|
|
// them, and make this the current resource operation so that everybody piles up on it.
|
|
if (serial) {
|
|
resourceChain = finalOp;
|
|
if (resourceChainLabel) {
|
|
log.debug(`Resource RPC serialization requested: ${label} is behind ${resourceChainLabel}`);
|
|
}
|
|
}
|
|
}
|