maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add Python resource transformations support (#3319)

Browse source 
Adds Python support for resource transformations aligned with the existing NodeJS support in #3174.

This PR also moves processing of transformations to earlier in the resource construction process (for both NodeJS and Python) to ensure that invariants established in the constructor cannot be violated by transformations. This change can technically be a breaking change, but given that (a) the transformations features was just released in 1.3.0 and (b) the cases where this is a breaking change are uncommon and unlikely to have been reliable anyway - it feels like a change we should make now.

Fixes #3283.
This commit is contained in:
Luke Hoban 2019-10-14 19:35:00 -05:00 committed by GitHub
parent 14da941b0c
commit 893e51d0ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 390 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CHANGELOG.md
View file

@ -31,6 +31,8 @@ CHANGELOG
will be used for any child resource of a component or stack.
[#3174](https://github.com/pulumi/pulumi/pull/3174)
- Add resource transformations support in Python. [#3319](https://github.com/pulumi/pulumi/pull/3319)
## 1.2.0 (2019-09-26)
- Support emitting high-level execution trace data to a file and add a debug-only command to view trace data.

25
sdk/nodejs/resource.ts
View file

@ -220,6 +220,27 @@ export abstract class Resource {
throw new ResourceError("Missing resource name argument (for URN creation)", opts.parent);
}
// Before anything else - if there are transformations registered, invoke them in order to transform the properties and
// options assigned to this resource.
const parent = opts.parent || getStackResource() || { __transformations: undefined };
this.__transformations = [ ...(opts.transformations || []), ...(parent.__transformations || []) ];
for (const transformation of this.__transformations) {
const tres = transformation({ resource: this, type: t, name, props, opts });
if (tres) {
if (tres.opts.parent !== opts.parent) {
// This is currently not allowed because the parent tree is needed to establish what
// transformation to apply in the first place, and to compute inheritance of other
// resource options in the Resource constructor before transformations are run (so
// modifying it here would only even partially take affect). It's theoretically
// possible this restriction could be lifted in the future, but for now just
// disallow re-parenting resources in transformations to be safe.
throw new Error("Transformations cannot currently be used to change the `parent` of a resource.");
}
props = tres.props;
opts = tres.opts;
}
}
this.__name = name;
// Make a shallow clone of opts to ensure we don't modify the value passed in.
@ -281,10 +302,6 @@ export abstract class Resource {
this.__providers = { ...this.__providers, ...providers };
}
// Combine transformations inherited from the parent with transformations provided in opts.
const parent = opts.parent || getStackResource() || { __transformations: undefined };
this.__transformations = [ ...(opts.transformations || []), ...(parent.__transformations || []) ];
this.__protect = !!opts.protect;
// Collapse any `Alias`es down to URNs. We have to wait until this point to do so because we do not know the

10
sdk/nodejs/runtime/resource.ts
View file

@ -163,16 +163,6 @@ export function registerResource(res: Resource, t: string, name: string, custom:
const label = `resource:${name}[${t}]`;
log.debug(`Registering resource: t=${t}, name=${name}, custom=${custom}`);
// If there are transformations registered, invoke them in order to transform the properties and
// options assigned to this resource.
for (const transformation of (res.__transformations || [])) {
const tres = transformation({ resource: res, type: t, name, props, opts });
if (tres) {
props = tres.props;
opts = tres.opts;
}
}
const monitor = getMonitor();
const resopAsync = prepareResource(label, res, custom, props, opts);

3
sdk/python/lib/pulumi/__init__.py
View file

@ -62,6 +62,9 @@ from .resource import (
create_urn,
export,
ROOT_STACK_RESOURCE,
ResourceTransformation,
ResourceTransformationArgs,
ResourceTransformationResult,
)
from .output import (

123
sdk/python/lib/pulumi/resource.py
View file

@ -13,7 +13,7 @@
# limitations under the License.
"""The Resource module, containing all resource-related definitions."""
from typing import Optional, List, Any, Mapping, Union, TYPE_CHECKING
from typing import Optional, List, Any, Mapping, Union, Callable, TYPE_CHECKING
import copy
@ -202,6 +202,80 @@ def collapse_alias_to_urn(
return Output.from_input(alias).apply(collapse_alias_to_urn_worker)
class ResourceTransformationArgs:
"""
ResourceTransformationArgs is the argument bag passed to a resource transformation.
"""
resource: 'Resource'
"""
The Resource instance that is being transformed.
"""
type_: str
"""
The type of the Resource.
"""
name: str
"""
The name of the Resource.
"""
props: 'Inputs'
"""
The original properties passed to the Resource constructor.
"""
opts: 'ResourceOptions'
"""
The original resource options passed to the Resource constructor.
"""
def __init__(self,
resource: 'Resource',
type_: str,
name: str,
props: 'Inputs',
opts: 'ResourceOptions') -> None:
self.resource = resource
self.type_ = type_
self.name = name
self.props = props
self.opts = opts
class ResourceTransformationResult:
"""
ResourceTransformationResult is the result that must be returned by a resource transformation
callback. It includes new values to use for the `props` and `opts` of the `Resource` in place of
the originally provided values.
"""
props: 'Inputs'
"""
The new properties to use in place of the original `props`.
"""
opts: 'ResourceOptions'
"""
The new resource options to use in place of the original `opts`
"""
def __init__(self,
props: 'Inputs',
opts: 'ResourceOptions') -> None:
self.props = props
self.opts = opts
ResourceTransformation = Callable[[ResourceTransformationArgs], Optional[ResourceTransformationResult]]
"""
ResourceTransformation is the callback signature for the `transformations` resource option. A
transformation is passed the same set of inputs provided to the `Resource` constructor, and can
optionally return back alternate values for the `props` and/or `opts` prior to the resource
actually being created. The effect will be as though those props and opts were passed in place
of the original call to the `Resource` constructor. If the transformation returns undefined,
this indicates that the resource will not be transformed.
"""
class ResourceOptions:
"""
@ -271,6 +345,13 @@ class ResourceOptions:
An optional customTimeouts config block.
"""
transformations: Optional[List[ResourceTransformation]]
"""
Optional list of transformations to apply to this resource during construction. The
transformations are applied in order, and are applied prior to transformation applied to
parents walking from the resource up to the stack.
"""
id: Optional['Input[str]']
"""
An optional existing ID to load, rather than create.
@ -298,7 +379,8 @@ class ResourceOptions:
additional_secret_outputs: Optional[List[str]] = None,
id: Optional['Input[str]'] = None,
import_: Optional[str] = None,
custom_timeouts: Optional['CustomTimeouts'] = None) -> None:
custom_timeouts: Optional['CustomTimeouts'] = None,
transformations: Optional[List[ResourceTransformation]] = None) -> None:
"""
:param Optional[Resource] parent: If provided, the currently-constructing resource should be the child of
the provided parent resource.
@ -315,12 +397,14 @@ class ResourceOptions:
or replacements.
:param Optional[List[string]] additional_secret_outputs: If provided, a list of output property names that should
also be treated as secret.
:param Optional[CustomTimeouts] customTimeouts: If provided, a config block for custom timeout information.
:param Optional[str] id: If provided, an existing resource ID to read, rather than create.
:param Optional[str] import_: When provided with a resource ID, import indicates that this resource's provider should
import its state from the cloud resource with the given ID. The inputs to the resource's constructor must align
with the resource's current state. Once a resource has been imported, the import property must be removed from
the resource's options.
:param Optional[CustomTimeouts] customTimeouts: If provided, a config block for custom timeout information.
:param Optional[transformations] transformations: If provided, a list of transformations to apply to this resource
during construction.
"""
# Expose 'merge' again this this object, but this time as an instance method.
@ -340,6 +424,7 @@ class ResourceOptions:
self.custom_timeouts = custom_timeouts
self.id = id
self.import_ = import_
self.transformations = transformations
if depends_on is not None:
for dep in depends_on:
@ -401,6 +486,7 @@ class ResourceOptions:
dest.ignore_changes = _merge_lists(dest.ignore_changes, source.ignore_changes)
dest.aliases = _merge_lists(dest.aliases, source.aliases)
dest.additional_secret_outputs = _merge_lists(dest.additional_secret_outputs, source.additional_secret_outputs)
dest.transformations = _merge_lists(dest.transformations, source.transformations)
dest.parent = dest.parent if source.parent is None else source.parent
dest.protect = dest.protect if source.protect is None else source.protect
@ -447,10 +533,10 @@ def _collapse_providers(opts: 'ResourceOptions'):
def _merge_lists(dest, source):
if dest is None:
dest = []
return source
if source is None:
source = []
return dest
return dest + source
@ -477,6 +563,11 @@ class Resource:
When set to true, protect ensures this resource cannot be deleted.
"""
_transformations: 'List[ResourceTransformation]'
"""
A collection of transformations to apply as part of resource registration.
"""
_aliases: 'Input[str]'
"""
A list of aliases applied to this resource.
@ -519,6 +610,28 @@ class Resource:
elif not isinstance(opts, ResourceOptions):
raise TypeError('Expected resource options to be a ResourceOptions instance')
# Before anything else - if there are transformations registered, give them a chance to run to modify the user provided
# properties and options assigned to this resource.
parent = opts.parent
if parent is None:
parent = get_root_resource()
parent_transformations = (parent._transformations or []) if parent is not None else []
self._transformations = (opts.transformations or []) + parent_transformations
for transformation in self._transformations:
args = ResourceTransformationArgs(resource=self, type_=t, name=name, props=props, opts=opts)
tres = transformation(args)
if tres is not None:
if tres.opts.parent != opts.parent:
# This is currently not allowed because the parent tree is needed to establish what
# transformation to apply in the first place, and to compute inheritance of other
# resource options in the Resource constructor before transformations are run (so
# modifying it here would only even partially take affect). It's theoretically
# possible this restriction could be lifted in the future, but for now just
# disallow re-parenting resources in transformations to be safe.
raise Exception("Transformations cannot currently be used to change the `parent` of a resource.")
props = tres.props
opts = tres.opts
self._name = name
# Make a shallow clone of opts to ensure we don't modify the value passed in.

1
sdk/python/lib/pulumi/runtime/__init__.py
View file

@ -32,6 +32,7 @@ from .settings import (
from .stack import (
run_in_stack,
get_root_resource,
register_stack_transformation,
)
from .invoke import (

14
sdk/python/lib/pulumi/runtime/stack.py
View file

@ -20,7 +20,7 @@ import collections
from inspect import isawaitable
from typing import Callable, Any, Dict, List
from ..resource import ComponentResource, Resource
from ..resource import ComponentResource, Resource, ResourceTransformation
from .settings import get_project, get_stack, get_root_resource, set_root_resource
from .rpc_manager import RPC_MANAGER
from .. import log
@ -186,3 +186,15 @@ def is_primitive(attr: Any) -> bool:
pass
return True
def register_stack_transformation(t: ResourceTransformation):
"""
Add a transformation to all future resources constructed in this Pulumi stack.
"""
root_resource = get_root_resource()
if root_resource is None:
raise Exception("The root stack resource was referenced before it was initialized.")
if root_resource._transformations is None:
root_resource._transformations = [t]
else:
root_resource._transformations = root_resource._transformations + [t]

1
tests/integration/transformations/nodejs/simple/index.ts
View file

@ -105,6 +105,7 @@ const res4 = new MyComponent("res4", {
],
});
// Scenario #5 - cross-resource transformations that inject dependencies on one resource into another.
class MyOtherComponent extends pulumi.ComponentResource {
child1: SimpleResource;
child2: SimpleResource;

3
tests/integration/transformations/python/simple/Pulumi.yaml Normal file
View file

@ -0,0 +1,3 @@
name: aliases_retype_component
description: A program that replaces a resource with a new name and alias.
runtime: python

138
tests/integration/transformations/python/simple/__main__.py Normal file
View file

@ -0,0 +1,138 @@
# Copyright 2016-2018, Pulumi Corporation. All rights reserved.
import asyncio
from pulumi import Output, ComponentResource, ResourceOptions, ResourceTransformationArgs, ResourceTransformationResult
from pulumi.dynamic import Resource, ResourceProvider, CreateResult
from pulumi.runtime import register_stack_transformation
class SimpleProvider(ResourceProvider):
def create(self, inputs):
return CreateResult("0", { "output": "a", "output2": "b" })
class SimpleResource(Resource):
output: Output[str]
output2: Output[str]
def __init__(self, name, args, opts = None):
super().__init__(SimpleProvider(),
name,
{ **args, "outputs": None, "output2": None },
opts)
class MyComponent(ComponentResource):
child: SimpleResource
def __init__(self, name, opts = None):
super().__init__("my:component:MyComponent", name, {}, opts)
childOpts = ResourceOptions(parent=self,
additional_secret_outputs=["output2"])
self.child = SimpleResource(f"{name}-child", { "input": "hello" }, childOpts)
self.register_outputs({})
# Scenario #1 - apply a transformation to a CustomResource
def res1_transformation(args: ResourceTransformationArgs):
print("res1 transformation")
return ResourceTransformationResult(
props=args.props,
opts=ResourceOptions.merge(args.opts, ResourceOptions(
additional_secret_outputs=["output"],
))
)
res1 = SimpleResource(
name="res1",
args={"input": "hello"},
opts=ResourceOptions(transformations=[res1_transformation]))
# Scenario #2 - apply a transformation to a Component to transform it's children
def res2_transformation(args: ResourceTransformationArgs):
print("res2 transformation")
if args.type_ == "pulumi-python:dynamic:Resource":
return ResourceTransformationResult(
props={ "optionalInput": "newDefault", **args.props },
opts=ResourceOptions.merge(args.opts, ResourceOptions(
additional_secret_outputs=["output"],
)))
res2 = MyComponent(
name="res2",
opts=ResourceOptions(transformations=[res2_transformation]))
# Scenario #3 - apply a transformation to the Stack to transform all (future) resources in the stack
def res3_transformation(args: ResourceTransformationArgs):
print("stack transformation")
if args.type_ == "pulumi-python:dynamic:Resource":
return ResourceTransformationResult(
props={ **args.props, "optionalInput": "stackDefault" },
opts=ResourceOptions.merge(args.opts, ResourceOptions(
additional_secret_outputs=["output"],
)))
register_stack_transformation(res3_transformation)
res3 = SimpleResource("res3", { "input": "hello" });
# Scenario #4 - transformations are applied in order of decreasing specificity
# 1. (not in this example) Child transformation
# 2. First parent transformation
# 3. Second parent transformation
# 4. Stack transformation
def res4_transformation_1(args: ResourceTransformationArgs):
print("res4 transformation")
if args.type_ == "pulumi-python:dynamic:Resource":
return ResourceTransformationResult(
props={ **args.props, "optionalInput": "default1" },
opts=args.opts)
def res4_transformation_2(args: ResourceTransformationArgs):
print("res4 transformation2")
if args.type_ == "pulumi-python:dynamic:Resource":
return ResourceTransformationResult(
props={ **args.props, "optionalInput": "default2" },
opts=args.opts)
res4 = MyComponent(
name="res4",
opts=ResourceOptions(transformations=[
res4_transformation_1,
res4_transformation_2]))
# Scenario #5 - cross-resource transformations that inject dependencies on one resource into another.
class MyOtherComponent(ComponentResource):
child1: SimpleResource
child2: SimpleResource
def __init__(self, name, opts = None):
super().__init__("my:component:MyComponent", name, {}, opts)
self.child = SimpleResource(f"{name}-child1", { "input": "hello" }, ResourceOptions(parent=self))
self.child = SimpleResource(f"{name}-child2", { "input": "hello" }, ResourceOptions(parent=self))
self.register_outputs({})
def transform_child1_depends_on_child2():
# Create a future that wil be resolved once we find child2. This is needed because we do not
# know what order we will see the resource registrations of child1 and child2.
child2_future = asyncio.Future()
def transform(args: ResourceTransformationArgs):
print("res4 transformation")
if args.name.endswith("-child2"):
# Resolve the child2 promise with the child2 resource.
child2_future.set_result(args.resource)
return None
elif args.name.endswith("-child1"):
# Overwrite the `input` to child2 with a dependency on the `output2` from child1.
async def getOutput2(input):
if input != "hello":
# Not strictly necessary - but shows we can confirm invariants we expect to be
# true.
raise Exception("unexpected input value")
child2 = await child2_future
return child2.output2
child2_input = Output.from_input(args.props["input"]).apply(getOutput2)
# Finally - overwrite the input of child2.
return ResourceTransformationResult(
props={ **args.props, "input": child2_input },
opts=args.opts)
return transform
res5 = MyOtherComponent(
name="res5",
opts=ResourceOptions(transformations=[transform_child1_depends_on_child2()]))

0
tests/integration/transformations/python/simple/requirements.txt Normal file
View file

160
tests/integration/transformations/transformations_test.go
View file

@ -4,6 +4,7 @@ package ints
import (
"path"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
@ -17,81 +18,100 @@ var dirs = []string{
"simple",
}
// TestNodejsAliases tests a case where a resource's name changes but it provides an `alias`
// pointing to the old URN to ensure the resource is preserved across the update.
func TestNodejsAliases(t *testing.T) {
func TestNodejsTransformations(t *testing.T) {
for _, dir := range dirs {
d := path.Join("nodejs", dir)
t.Run(d, func(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: d,
Dependencies: []string{"@pulumi/pulumi"},
Quick: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
foundRes1 := false
foundRes2Child := false
foundRes3 := false
foundRes4Child := false
foundRes5Child := false
for _, res := range stack.Deployment.Resources {
// "res1" has a transformation which adds additionalSecretOutputs
if res.URN.Name() == "res1" {
foundRes1 = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
}
// "res2" has a transformation which adds additionalSecretOutputs to it's
// "child"
if res.URN.Name() == "res2-child" {
foundRes2Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output2"))
}
// "res3" is impacted by a global stack transformation which sets
// optionalDefault to "stackDefault"
if res.URN.Name() == "res3" {
foundRes3 = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res4" is impacted by two component parent transformations which set
// optionalDefault to "default1" and then "default2" and also a global stack
// transformation which sets optionalDefault to "stackDefault". The end
// result should be "stackDefault".
if res.URN.Name() == "res4-child" {
foundRes4Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res5" modifies one of its children to depend on another of its children.
if res.URN.Name() == "res5-child1" {
foundRes5Child = true
assert.Equal(t, res.Type, tokens.Type("pulumi-nodejs:dynamic:Resource"))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
// TODO[pulumi/pulumi#3282] Due to this bug, the dependency information
// will not be correctly recorded in the state file, and so cannot be
// verified here.
//
// assert.Len(t, res.PropertyDependencies, 1)
input := res.Inputs["input"]
assert.NotNil(t, input)
assert.Equal(t, "b", input.(string))
}
}
assert.True(t, foundRes1)
assert.True(t, foundRes2Child)
assert.True(t, foundRes3)
assert.True(t, foundRes4Child)
assert.True(t, foundRes5Child)
},
Dir: d,
Dependencies: []string{"@pulumi/pulumi"},
Quick: true,
ExtraRuntimeValidation: Validator("nodejs"),
})
})
}
}
func TestPythonTransformations(t *testing.T) {
for _, dir := range dirs {
d := path.Join("python", dir)
t.Run(d, func(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: d,
Dependencies: []string{
filepath.Join("..", "..", "..", "sdk", "python", "env", "src"),
},
Quick: true,
ExtraRuntimeValidation: Validator("python"),
})
})
}
}
func Validator(language string) func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
dynamicResName := "pulumi-" + language + ":dynamic:Resource"
return func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
foundRes1 := false
foundRes2Child := false
foundRes3 := false
foundRes4Child := false
foundRes5Child := false
for _, res := range stack.Deployment.Resources {
// "res1" has a transformation which adds additionalSecretOutputs
if res.URN.Name() == "res1" {
foundRes1 = true
assert.Equal(t, res.Type, tokens.Type(dynamicResName))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
}
// "res2" has a transformation which adds additionalSecretOutputs to it's
// "child"
if res.URN.Name() == "res2-child" {
foundRes2Child = true
assert.Equal(t, res.Type, tokens.Type(dynamicResName))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output"))
assert.Contains(t, res.AdditionalSecretOutputs, resource.PropertyKey("output2"))
}
// "res3" is impacted by a global stack transformation which sets
// optionalDefault to "stackDefault"
if res.URN.Name() == "res3" {
foundRes3 = true
assert.Equal(t, res.Type, tokens.Type(dynamicResName))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res4" is impacted by two component parent transformations which set
// optionalDefault to "default1" and then "default2" and also a global stack
// transformation which sets optionalDefault to "stackDefault". The end
// result should be "stackDefault".
if res.URN.Name() == "res4-child" {
foundRes4Child = true
assert.Equal(t, res.Type, tokens.Type(dynamicResName))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
optionalInput := res.Inputs["optionalInput"]
assert.NotNil(t, optionalInput)
assert.Equal(t, "stackDefault", optionalInput.(string))
}
// "res5" modifies one of its children to depend on another of its children.
if res.URN.Name() == "res5-child1" {
foundRes5Child = true
assert.Equal(t, res.Type, tokens.Type(dynamicResName))
assert.Equal(t, res.Parent.Type(), tokens.Type("my:component:MyComponent"))
// TODO[pulumi/pulumi#3282] Due to this bug, the dependency information
// will not be correctly recorded in the state file, and so cannot be
// verified here.
//
// assert.Len(t, res.PropertyDependencies, 1)
input := res.Inputs["input"]
assert.NotNil(t, input)
assert.Equal(t, "b", input.(string))
}
}
assert.True(t, foundRes1)
assert.True(t, foundRes2Child)
assert.True(t, foundRes3)
assert.True(t, foundRes4Child)
assert.True(t, foundRes5Child)
}
}