maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Transformations (#3174)

Browse source 
Adds the ability to provide `transformations` to modify the properties and resource options that will be used for any child resource of a component or stack.

This offers an "escape hatch" to modify the behaviour of a component by peeking behind it's abstraction.  For example, it can be used to add a resource option (`additionalSecretOutputs`, `aliases`, `protect`, etc.) to a specific known child of a component, or to modify some input property to a child resource if the component does not (yet) expose the ability to control that input directly.  It could also be used for more interesting scenarios - such as:
1. Automatically applying tags to all resources that support them in a stack (or component)
2. Injecting real dependencies between stringly-referenced  resources in a Helm Chart 
3. Injecting explicit names using a preferred naming convention across all resources in a stack
4. Injecting `import` onto all resources by doing a lookup into a name=>id mapping

Because this feature makes it possible to peek behind a component abstraction, it must be used with care in cases where the component is versioned independently of the use of transformations.  Also, this can result in "spooky action at a distance", so should be used judiciously.  That said - this can be used as an escape hatch to unblock a wide variety of common use cases without waiting on changes to be made in a component implementation.  

Each transformation is passed the `resource`, `name`, `type`, `props` and `opts` that are passed into the `Resource` constructor for any resource descended from the resource that has the transformation applied.  The transformation callback can optionally return alternate versions of the `props` and `opts` to be used in place of the original values provided to the resource constructor.

Fixes #2068.
This commit is contained in:
Luke Hoban 2019-09-29 11:27:37 -07:00 committed by GitHub
parent ec85408eee
commit 9374c374c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 409 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -18,6 +18,9 @@ CHANGELOG
[#3183](https://github.com/pulumi/pulumi/pull/3183)
- Add Codefresh CI detection.
- Add `-c` (config array) flag to the `preview` command.
- Adds the ability to provide transformations to modify the properties and resource options that
will be used for any child resource of a component or stack.
[#3174](https://github.com/pulumi/pulumi/pull/3174)
## 1.1.0 (2019-09-11)
@ -48,6 +51,7 @@ CHANGELOG
- Filter the list of templates shown by default during `pulumi new`.
[#3147](https://github.com/pulumi/pulumi/pull/3147)
## 1.0.0-beta.4 (2019-08-22)
- Fix a crash when using StackReference from the `1.0.0-beta.3` version of

16
pkg/resource/deploy/step_generator.go
View file

@ -341,15 +341,13 @@ func (sg *stepGenerator) GenerateSteps(event RegisterResourceEvent) ([]Step, res
if hasOld {
contract.Assert(old != nil)
diff, err := sg.diff(urn, old, new, oldInputs, oldOutputs, inputs, prov, allowUnknowns, goal.IgnoreChanges)
if err != nil {
// If the plugin indicated that the diff is unavailable, assume that the resource will be updated and
// report the message contained in the error.
if _, ok := err.(plugin.DiffUnavailableError); ok {
diff = plugin.DiffResult{Changes: plugin.DiffSome}
sg.plan.ctx.Diag.Warningf(diag.RawMessage(urn, err.Error()))
} else {
return nil, result.FromError(err)
}
// If the plugin indicated that the diff is unavailable, assume that the resource will be updated and
// report the message contained in the error.
if _, ok := err.(plugin.DiffUnavailableError); ok {
diff = plugin.DiffResult{Changes: plugin.DiffSome}
sg.plan.ctx.Diag.Warningf(diag.RawMessage(urn, err.Error()))
} else if err != nil {
return nil, result.FromError(err)
}
// Ensure that we received a sensible response.

78
sdk/nodejs/resource.ts
View file

@ -15,6 +15,7 @@
import { util } from "protobufjs";
import { ResourceError, RunError } from "./errors";
import { all, Input, Inputs, interpolate, Output, output } from "./output";
import { getStackResource } from "./runtime";
import { readResource, registerResource, registerResourceOutputs } from "./runtime/resource";
import { getProject, getStack } from "./runtime/settings";
import * as utils from "./utils";
@ -140,12 +141,23 @@ export abstract class Resource {
// tslint:disable-next-line:variable-name
private readonly __protect: boolean;
/**
* @internal
* A collection of transformations to apply as part of resource registration.
*
* Note: This is marked optional only because older versions of this library may not have had
* this property, and marking optional forces consumers of the property to defensively handle
* cases where they are passed "old" resources.
*/
// tslint:disable-next-line:variable-name
__transformations?: ResourceTransformation[];
/**
* @internal
* A list of aliases applied to this resource.
*
* Note: This is marked optional only because older versions of this library may not have had
* this property, and marking optional forces conumers of the property to defensively handle
* this property, and marking optional forces consumers of the property to defensively handle
* cases where they are passed "old" resources.
*/
// tslint:disable-next-line:variable-name
@ -156,7 +168,7 @@ export abstract class Resource {
* The name assigned to the resource at construction.
*
* Note: This is marked optional only because older versions of this library may not have had
* this property, and marking optional forces conumers of the property to defensively handle
* this property, and marking optional forces consumers of the property to defensively handle
* cases where they are passed "old" resources.
*/
// tslint:disable-next-line:variable-name
@ -269,6 +281,10 @@ export abstract class Resource {
this.__providers = { ...this.__providers, ...providers };
}
// Combine transformations inherited from the parent with transformations provided in opts.
const parent = opts.parent || getStackResource() || { __transformations: undefined };
this.__transformations = [ ...(opts.transformations || []), ...(parent.__transformations || []) ];
this.__protect = !!opts.protect;
// Collapse any `Alias`es down to URNs. We have to wait until this point to do so because we do not know the
@ -466,6 +482,12 @@ export interface ResourceOptions {
* An optional customTimeouts configuration block.
*/
customTimeouts?: CustomTimeouts;
/**
* Optional list of transformations to apply to this resource during construction. The
* transformations are applied in order, and are applied prior to transformation applied to
* parents walking from the resource up to the stack.
*/
transformations?: ResourceTransformation[];
// !!! IMPORTANT !!! If you add a new field to this type, make sure to add test that verifies
// that mergeOptions works properly for it.
@ -486,6 +508,58 @@ export interface CustomTimeouts {
delete?: string;
}
/**
* ResourceTransformation is the callback signature for the `transformations` resource option. A
* transformation is passed the same set of inputs provided to the `Resource` constructor, and can
* optionally return back alternate values for the `props` and/or `opts` prior to the resource
* actually being created. The effect will be as though those props and opts were passed in place
* of the original call to the `Resource` constructor. If the transformation returns undefined,
* this indicates that the resource will not be transformed.
*/
export type ResourceTransformation = (args: ResourceTransformationArgs) => ResourceTransformationResult | undefined;
/**
* ResourceTransformationArgs is the argument bag passed to a resource transformation.
*/
export interface ResourceTransformationArgs {
/**
* The Resource instance that is being transformed.
*/
resource: Resource;
/**
* The type of the Resource.
*/
type: string;
/**
* The name of the Resource.
*/
name: string;
/**
* The original properties passed to the Resource constructor.
*/
props: Inputs;
/**
* The original resource options passed to the Resource constructor.
*/
opts: ResourceOptions;
}
/**
* ResourceTransformationResult is the result that must be returned by a resource transformation
* callback. It includes new values to use for the `props` and `opts` of the `Resource` in place of
* the originally provided values.
*/
export interface ResourceTransformationResult {
/**
* The new properties to use in place of the original `props`
*/
props: Inputs;
/**
* The new resource options to use in place of the original `opts`
*/
opts: ResourceOptions;
}
/**
* CustomResourceOptions is a bag of optional settings that control a custom resource's behavior.
*/

10
sdk/nodejs/runtime/resource.ts
View file

@ -163,6 +163,16 @@ export function registerResource(res: Resource, t: string, name: string, custom:
const label = `resource:${name}[${t}]`;
log.debug(`Registering resource: t=${t}, name=${name}, custom=${custom}`);
// If there are transformations registered, invoke them in order to transform the properties and
// options assigned to this resource.
for (const transformation of (res.__transformations || [])) {
const tres = transformation({ resource: res, type: t, name, props, opts });
if (tres) {
props = tres.props;
opts = tres.opts;
}
}
const monitor = getMonitor();
const resopAsync = prepareResource(label, res, custom, props, opts);

24
sdk/nodejs/runtime/stack.ts
View file

@ -15,7 +15,7 @@
import * as asset from "../asset";
import { getProject, getStack } from "../metadata";
import { Inputs, Output, output, secret } from "../output";
import { ComponentResource, Resource } from "../resource";
import { ComponentResource, Resource, ResourceTransformation } from "../resource";
import { getRootResource, isQueryMode, setRootResource } from "./settings";
/**
@ -25,6 +25,13 @@ import { getRootResource, isQueryMode, setRootResource } from "./settings";
*/
export const rootPulumiStackTypeName = "pulumi:pulumi:Stack";
let stackResource: Stack | undefined;
// Get the root stack resource for the current stack deployment
export function getStackResource(): Stack | undefined {
return stackResource;
}
/**
* runInPulumiStack creates a new Pulumi stack resource and executes the callback inside of it. Any outputs
* returned by the callback will be stored as output properties on this resulting Stack object.
@ -64,8 +71,11 @@ class Stack extends ComponentResource {
if (parent) {
throw new Error("Only one root Pulumi Stack may be active at once");
}
await setRootResource(this);
// Set the global reference to the stack resource before invoking this init() function
stackResource = this;
let outputs: Inputs | undefined;
try {
outputs = await massage(init(), []);
@ -191,3 +201,13 @@ async function massageComplex(prop: any, objectStack: any[]): Promise<any> {
return obj;
}
}
/**
* Add a transformation to all future resources constructed in this Pulumi stack.
*/
export function registerStackTransformation(t: ResourceTransformation) {
if (!stackResource) {
throw new Error("The root stack resource was referenced before it was initialized.");
}
stackResource.__transformations = [...(stackResource.__transformations || []), t];
}

25
sdk/nodejs/tests/options.spec.ts
View file

@ -103,6 +103,31 @@ describe("options", () => {
});
});
describe("arrayTransformations", () => {
const a = () => undefined;
const b = () => undefined;
it("keeps value from opts1 if not provided in opts2", asyncTest(async () => {
const result = mergeOptions({ transformations: [a] }, {});
assert.deepStrictEqual(result.transformations, [a]);
}));
it("keeps value from opts2 if not provided in opts1", asyncTest(async () => {
const result = mergeOptions({ }, { transformations: [a] });
assert.deepStrictEqual(result.transformations, [a]);
}));
it("does nothing to value from opts1 if given null in opts2", asyncTest(async () => {
const result = mergeOptions({ transformations: [a] }, { transformations: null! });
assert.deepStrictEqual(result.transformations, [a]);
}));
it("does nothing to value from opts1 if given undefined in opts2", asyncTest(async () => {
const result = mergeOptions({ transformations: [a] }, { transformations: undefined });
assert.deepStrictEqual(result.transformations, [a]);
}));
it("merges values from opts1 if given value in opts2", asyncTest(async () => {
const result = mergeOptions({ transformations: [a] }, { transformations: [b] });
assert.deepStrictEqual(result.transformations, [a, b]);
}));
});
describe("providers", () => {
const awsProvider = <ProviderResource>{ getPackage: () => "aws" };
const azureProvider = <ProviderResource>{ getPackage: () => "azure" };

3
tests/integration/transformations/nodejs/simple/Pulumi.yaml Normal file
View file

@ -0,0 +1,3 @@
name: transformations_simple
description:
runtime: nodejs

153
tests/integration/transformations/nodejs/simple/index.ts Normal file
View file

@ -0,0 +1,153 @@
// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
import * as pulumi from "@pulumi/pulumi";
const simpleProvider: pulumi.dynamic.ResourceProvider = {
async create(inputs: any) {
return {
id: "0",
outs: { output: "a", output2: "b" },
};
},
};
interface SimpleArgs {
input: pulumi.Input<string>;
optionalInput?: pulumi.Input<string>;
}
class SimpleResource extends pulumi.dynamic.Resource {
output: pulumi.Output<string>;
output2: pulumi.Output<string>;
constructor(name, args: SimpleArgs, opts?: pulumi.CustomResourceOptions) {
super(simpleProvider, name, { ...args, output: undefined, output2: undefined }, opts);
}
}
class MyComponent extends pulumi.ComponentResource {
child: SimpleResource;
constructor(name: string, opts?: pulumi.ComponentResourceOptions) {
super("my:component:MyComponent", name, {}, opts);
this.child = new SimpleResource(`${name}-child`, { input: "hello" }, {
parent: this,
additionalSecretOutputs: ["output2"],
});
this.registerOutputs({});
}
}
// Scenario #1 - apply a transformation to a CustomResource
const res1 = new SimpleResource("res1", { input: "hello" }, {
transformations: [
({ props, opts }) => {
console.log("res1 transformation");
return {
props: props,
opts: pulumi.mergeOptions(opts, { additionalSecretOutputs: ["output"] }),
};
},
],
});
// Scenario #2 - apply a transformation to a Component to transform it's children
const res2 = new MyComponent("res2", {
transformations: [
({ type, props, opts }) => {
console.log("res2 transformation");
if (type === "pulumi-nodejs:dynamic:Resource") {
return {
props: { optionalInput: "newDefault", ...props },
opts: pulumi.mergeOptions(opts, { additionalSecretOutputs: ["output"] }),
};
}
},
],
});
// Scenario #3 - apply a transformation to the Stack to transform all (future) resources in the stack
pulumi.runtime.registerStackTransformation(({ type, props, opts }) => {
console.log("stack transformation");
if (type === "pulumi-nodejs:dynamic:Resource") {
return {
props: { ...props, optionalInput: "stackDefault" },
opts: pulumi.mergeOptions(opts, { additionalSecretOutputs: ["output"] }),
};
}
});
const res3 = new SimpleResource("res3", { input: "hello" });
// Scenario #4 - transformations are applied in order of decreasing specificity
// 1. (not in this example) Child transformation
// 2. First parent transformation
// 3. Second parent transformation
// 4. Stack transformation
const res4 = new MyComponent("res4", {
transformations: [
({ type, props, opts }) => {
console.log("res4 transformation");
if (type === "pulumi-nodejs:dynamic:Resource") {
return {
props: { ...props, optionalInput: "default1" },
opts,
};
}
},
({ type, props, opts }) => {
console.log("res4 transformation 2");
if (type === "pulumi-nodejs:dynamic:Resource") {
return {
props: { ...props, optionalInput: "default2" },
opts,
};
}
},
],
});
class MyOtherComponent extends pulumi.ComponentResource {
child1: SimpleResource;
child2: SimpleResource;
constructor(name: string, opts?: pulumi.ComponentResourceOptions) {
super("my:component:MyComponent", name, {}, opts);
this.child1 = new SimpleResource(`${name}-child1`, { input: "hello" }, { parent: this });
this.child2 = new SimpleResource(`${name}-child2`, { input: "hello" }, { parent: this });
this.registerOutputs({});
}
}
const transformChild1DependsOnChild2: pulumi.ResourceTransformation = (() => {
// Create a promise that wil be resolved once we find child2. This is needed because we do not
// know what order we will see the resource registrations of child1 and child2.
let child2Found: (res: pulumi.Resource) => void;
const child2 = new Promise<pulumi.Resource>((res) => child2Found = res);
// Return a transformation which will rewrite child1 to depend on the promise for child2, and
// will resolve that promise when it finds child2.
return (args: pulumi.ResourceTransformationArgs) => {
if (args.name.endsWith("-child2")) {
// Resolve the child2 promise with the child2 resource.
child2Found(args.resource);
return undefined;
} else if (args.name.endsWith("-child1")) {
// Overwrite the `input` to child2 with a dependency on the `output2` from child1.
const child2Input = pulumi.output(args.props["input"]).apply(async (input) => {
if (input !== "hello") {
// Not strictly necessary - but shows we can confirm invariants we expect to be
// true.
throw new Error("unexpected input value");
}
return child2.then(c2Res => c2Res["output2"]);
});
// Finally - overwrite the input of child2.
return {
props: { ...args.props, input: child2Input },
opts: args.opts,
};
}
};
})();
const res5 = new MyOtherComponent("res5", {
transformations: [ transformChild1DependsOnChild2 ],
});

12
tests/integration/transformations/nodejs/simple/package.json Normal file
View file

@ -0,0 +1,12 @@
{
"name": "aliases",
"license": "Apache-2.0",
"main": "bin/index.js",
"typings": "bin/index.d.ts",
"devDependencies": {
"typescript": "^2.5.3"
},
"peerDependencies": {
"@pulumi/pulumi": "latest"
}
}

97
tests/integration/transformations/transformations_test.go Normal file
View file

@ -0,0 +1,97 @@
// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
package ints
import (
"path"
"testing"
"github.com/stretchr/testify/assert"
"github.com/pulumi/pulumi/pkg/resource"
"github.com/pulumi/pulumi/pkg/testing/integration"
"github.com/pulumi/pulumi/pkg/tokens"
)
var dirs = []string{
"simple",
}
// TestNodejsAliases tests a case where a resource's name changes but it provides an `alias`
// pointing to the old URN to ensure the resource is preserved across the update.
func TestNodejsAliases(t *testing.T) {
for _, dir := range dirs {
d := path.Join("nodejs", dir)
t.Run(d, func(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: d,
Dependencies: []string{"@pulumi/pulumi"},
Quick: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
foundRes1 := false
foundRes2Child := false
foundRes3 := false
foundRes4Child := false
foundRes5Child := false
for _, res := range stack.Deployment.Resources {
// "res1" has a transformation which adds additionalSecretOutputs
if res.URN.Name() == "res1" {
foundRes1 = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
}
// "res2" has a transformation which adds additionalSecretOutputs to it's
// "child"
if res.URN.Name() == "res2-child" {
foundRes2Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output2"))
}
// "res3" is impacted by a global stack transformation which sets
// optionalDefault to "stackDefault"
if res.URN.Name() == "res3" {
foundRes3 = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res4" is impacted by two component parent transformations which set
// optionalDefault to "default1" and then "default2" and also a global stack
// transformation which sets optionalDefault to "stackDefault". The end
// result should be "stackDefault".
if res.URN.Name() == "res4-child" {
foundRes4Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res5" modifies one of its children to depend on another of its children.
if res.URN.Name() == "res5-child1" {
foundRes5Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
// TODO[pulumi/pulumi#3282] Due to this bug, the dependency information
// will not be correctly recorded in the state file, and so cannot be
// verified here.
//
// assert.Len(t, res.PropertyDependencies, 1)
input := res.Inputs["input"]
assert.NotNil(t, input)
assert.Equal(t, "b", input.(string))
}
}
assert.True(t, foundRes1)
assert.True(t, foundRes2Child)
assert.True(t, foundRes3)
assert.True(t, foundRes4Child)
assert.True(t, foundRes5Child)
},
})
})
}
}