Implementation of Read for Python (#2752)

This commit implements read_resource functionality for Python in a
manner identical to the NodeJS implementation. If an "id" option is
passed to a resource via ResourceOptions on construction, that resource
will be read and not created.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Improvements
 
+- Pulumi now allows Python programs to "read" existing resources instead of just creating them. This feature enables
+  Pulumi Python packages to expose ".get()" methods that allow for reading of resources that already exist.
+
 ## 0.17.14 (Released May 28, 2019)
 
 ### Improvements

sdk/python/lib/pulumi/resource.py

@@ -16,7 +16,7 @@
 from typing import Optional, List, Any, Mapping, TYPE_CHECKING
 
 from .runtime import known_types
-from .runtime.resource import register_resource, register_resource_outputs
+from .runtime.resource import register_resource, register_resource_outputs, read_resource
 from .runtime.settings import get_root_resource
 
 if TYPE_CHECKING:
@@ -79,7 +79,12 @@ class ResourceOptions:
     to mark certain ouputs as a secrets on a per resource basis.
     """
 
+    id: Optional[str]
+    """
+    An optional existing ID to load, rather than create.
+    """
 
+    # pylint: disable=redefined-builtin
     def __init__(self,
                  parent: Optional['Resource'] = None,
                  depends_on: Optional[List['Resource']] = None,
@@ -89,7 +94,8 @@ class ResourceOptions:
                  delete_before_replace: Optional[bool] = None,
                  ignore_changes: Optional[List[str]] = None,
                  version: Optional[str] = None,
-                 additional_secret_outputs: Optional[List[str]] = None) -> None:
+                 additional_secret_outputs: Optional[List[str]] = None,
+                 id: Optional[str] = None) -> None:
         """
         :param Optional[Resource] parent: If provided, the currently-constructing resource should be the child of
                the provided parent resource.
@@ -106,6 +112,7 @@ class ResourceOptions:
                or replacements.
         :param Optional[List[string]] additional_secret_outputs: If provided, a list of output property names that should
                also be treated as secret.
+        :param Optional[str] id: If provided, an existing resource ID to read, rather than create.
         """
         self.parent = parent
         self.depends_on = depends_on
@@ -116,6 +123,7 @@ class ResourceOptions:
         self.ignore_changes = ignore_changes
         self.version = version
         self.additional_secret_outputs = additional_secret_outputs
+        self.id = id
 
         if depends_on is not None:
             for dep in depends_on:
@@ -199,7 +207,14 @@ class Resource:
                 self._providers = {**self._providers, **providers}
 
         self._protect = bool(opts.protect)
-        register_resource(self, t, name, custom, props, opts)
+
+        if opts.id is not None:
+            # If this resource already exists, read its state rather than registering it anow.
+            if not custom:
+                raise Exception("Cannot read an existing resource unless it has a custom provider")
+            read_resource(self, t, name, props, opts)
+        else:
+            register_resource(self, t, name, custom, props, opts)
 
     def translate_output_property(self, prop: str) -> str:
         """

sdk/python/lib/pulumi/runtime/resource.py

@@ -118,6 +118,117 @@ async def prepare_resource(res: 'Resource',
         property_dependencies
     )
 
+# pylint: disable=too-many-locals,too-many-statements
+def read_resource(res: 'Resource', ty: str, name: str, props: 'Inputs', opts: Optional['ResourceOptions']):
+    if opts.id is None:
+        raise Exception("Cannot read resource whose options are lacking an ID value")
+
+    log.debug(f"reading resource: ty={ty}, name={name}, id={opts.id}")
+    monitor = settings.get_monitor()
+
+    # Prepare the resource, similar to a RegisterResource. Reads are deliberately similar to RegisterResource except
+    # that we are populating the Resource object with properties associated with an already-live resource.
+    #
+    # Same as below, we initialize the URN property on the resource, which will always be resolved.
+    log.debug(f"preparing read resource for RPC")
+    urn_future = asyncio.Future()
+    urn_known = asyncio.Future()
+    urn_secret = asyncio.Future()
+    urn_known.set_result(True)
+    urn_secret.set_result(False)
+    resolve_urn = urn_future.set_result
+    resolve_urn_exn = urn_future.set_exception
+    res.urn = known_types.new_output({res}, urn_future, urn_known, urn_secret)
+
+    # Furthermore, since resources being Read must always be custom resources (enforced in the Resource constructor),
+    # we'll need to set up the ID field which will be populated at the end of the ReadResource call.
+    #
+    # Note that we technically already have the ID (opts.id), but it's more consistent with the rest of the model to
+    # resolve it asynchronously along with all of the other resources.
+    resolve_value = asyncio.Future()
+    resolve_perform_apply = asyncio.Future()
+    resolve_secret = asyncio.Future()
+    res.id = known_types.new_output({res}, resolve_value, resolve_perform_apply, resolve_secret)
+
+    def do_resolve(value: Any, perform_apply: bool, exn: Optional[Exception]):
+        if exn is not None:
+            resolve_value.set_exception(exn)
+            resolve_perform_apply.set_exception(exn)
+            resolve_secret.set_exception(exn)
+        else:
+            resolve_value.set_result(value)
+            resolve_perform_apply.set_result(perform_apply)
+            resolve_secret.set_result(False)
+
+    resolve_id = do_resolve
+
+    # Like below, "transfer" all input properties onto unresolved futures on res.
+    resolvers = rpc.transfer_properties(res, props)
+
+    async def do_read():
+        try:
+            log.debug(f"preparing read: ty={ty}, name={name}, id={opts.id}")
+            resolver = await prepare_resource(res, ty, True, props, opts)
+
+            # Resolve the ID that we were given. Note that we are explicitly discarding the list of dependencies
+            # returned to us from "serialize_property" (the second argument). This is because a "read" resource does
+            # not actually have any dependencies at all in the cloud provider sense, because a read resource already
+            # exists. We do not need to track this dependency.
+            resolved_id = await rpc.serialize_property(opts.id, [])
+            log.debug(f"read prepared: ty={ty}, name={name}, id={opts.id}")
+
+            # These inputs will end up in the snapshot, so if there are any additional secret outputs, record them
+            # here.
+            additional_secret_outputs = opts.additional_secret_outputs
+            if res.translate_input_property is not None and opts.additional_secret_outputs is not None:
+                additional_secret_outputs = map(res.translate_input_property, opts.additional_secret_outputs)
+
+            req = resource_pb2.ReadResourceRequest(
+                type=ty,
+                name=name,
+                id=resolved_id,
+                parent=resolver.parent_urn,
+                provider=resolver.provider_ref,
+                properties=resolver.serialized_props,
+                dependencies=resolver.dependencies,
+                version=opts.version or "",
+                acceptSecrets=True,
+                additionalSecretOutputs=additional_secret_outputs,
+            )
+
+            def do_rpc_call():
+                if monitor:
+                    # If there is a monitor available, make the true RPC request to the engine.
+                    try:
+                        return monitor.ReadResource(req)
+                    except grpc.RpcError as exn:
+                        # See the comment on invoke for the justification for disabling
+                        # this warning
+                        # pylint: disable=no-member
+                        if exn.code() == grpc.StatusCode.UNAVAILABLE:
+                            sys.exit(0)
+
+                        details = exn.details()
+                    raise Exception(details)
+                else:
+                    # If no monitor is available, we'll need to fake up a response, for testing.
+                    return RegisterResponse(create_test_urn(ty, name), None, resolver.serialized_props)
+
+            resp = await asyncio.get_event_loop().run_in_executor(None, do_rpc_call)
+
+        except Exception as exn:
+            log.debug(f"exception when preparing or executing rpc: {traceback.format_exc()}")
+            rpc.resolve_outputs_due_to_exception(resolvers, exn)
+            resolve_urn_exn(exn)
+            resolve_id(None, False, exn)
+            raise
+
+        log.debug(f"resource read successful: ty={ty}, urn={resp.urn}")
+        resolve_urn(resp.urn)
+        resolve_id(resolved_id, True, None) # Read IDs are always known.
+        await rpc.resolve_outputs(res, props, resp.properties, resolvers)
+
+    asyncio.ensure_future(RPC_MANAGER.do_rpc("read resource", do_read)())
 
 # pylint: disable=too-many-locals,too-many-statements
 def register_resource(res: 'Resource', ty: str, name: str, custom: bool, props: 'Inputs', opts: Optional['ResourceOptions']):
@@ -139,7 +250,7 @@ def register_resource(res: 'Resource', ty: str, name: str, custom: bool, props:
     urn_known = asyncio.Future()
     urn_secret = asyncio.Future()
     urn_known.set_result(True)
-    urn_secret.set_result(True)
+    urn_secret.set_result(False)
     resolve_urn = urn_future.set_result
     resolve_urn_exn = urn_future.set_exception
     res.urn = known_types.new_output({res}, urn_future, urn_known, urn_secret)

sdk/python/lib/test/langhost/read/__main__.py

@@ -0,0 +1,27 @@
+# Copyright 2016-2019, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pulumi import CustomResource, ResourceOptions
+
+CustomResource("test:read:resource", "foo", {
+    "a": "bar",
+    "b": ["c", 4, "d"],
+    "c": {
+        "nest": "baz"
+    }
+}, opts=ResourceOptions(id="myresourceid", version="0.17.9"))
+
+parent = CustomResource("test:index:MyResource", "foo2")
+CustomResource("test:read:resource", "foo-with-parent", {
+    "state": "foo",
+}, opts=ResourceOptions(id="myresourceid2", version="0.17.9", parent=parent))

sdk/python/lib/test/langhost/read/test_read.py

@@ -0,0 +1,56 @@
+# Copyright 2016-2019, Pulumi Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from os import path
+from ..util import LanghostTest
+
+
+class ReadTest(LanghostTest):
+    def test_read(self):
+        self.run_test(
+            program=path.join(self.base_path(), "read"),
+            expected_resource_count=1)
+
+    def register_resource(self, _ctx, _dry_run, ty, name, _resource,
+                          _dependencies, _parent, _custom, _protect, _provider, _property_deps, _delete_before_replace,
+                          _ignore_changes, _version):
+        self.assertEqual(ty, "test:index:MyResource")
+        self.assertEqual(name, "foo2")
+        return {
+            "urn": self.make_urn(ty, name),
+        }
+
+    def read_resource(self, ctx, ty, name, id, parent, state, dependencies, provider, version):
+        if name == "foo":
+            self.assertDictEqual(state, {
+                "a": "bar",
+                "b": ["c", 4, "d"],
+                "c": {
+                    "nest": "baz"
+                }
+            })
+            self.assertEqual(ty, "test:read:resource")
+            self.assertEqual(id, "myresourceid")
+            self.assertEqual(version, "0.17.9")
+        elif name == "foo-with-parent":
+            self.assertDictEqual(state, {
+                "state": "foo",
+            })
+            self.assertEqual(ty, "test:read:resource")
+            self.assertEqual(id, "myresourceid2")
+            self.assertEqual(parent, self.make_urn("test:index:MyResource", "foo2"))
+            self.assertEqual(version, "0.17.9")
+        return {
+            "urn": self.make_urn(ty, name),
+            "properties": state,
+        }

sdk/python/lib/test/langhost/util.py

@@ -68,8 +68,11 @@ class LanghostMockResourceMonitor(proto.ResourceMonitorServicer):
         id_ = request.id
         parent = request.parent
         state = rpc.deserialize_properties(request.properties)
+        dependencies = sorted(list(request.dependencies))
+        provider = request.provider
+        version = request.version
         outs = self.langhost_test.read_resource(context, type_, name, id_,
-                                                parent, state)
+                                                parent, state, dependencies, provider, version)
         if "properties" in outs:
             loop = asyncio.new_event_loop()
             props_proto = loop.run_until_complete(rpc.serialize_properties(outs["properties"], {}))