Implement more of the Python runtime

This change includes a lot more functionality.  Enough to actually
run the webserver-py example through previews, updates, and destroys!

* Actually wire up the gRPC connections to the engine/monitor.

* Move the Node.js and Python generated Protobuf/gRPC files underneath
  the actual SDK directories to simplify this generally.  No more
  copying during `make` and, in fact, this was required to give a smoother
  experience with good packages/modules for the Python's SDK development.

* Build the Python egg during `make build`.

* Add support for program stacks.  Just like with the Node.js runtime,
  we will auto-parent any resources without explicit parents to a single
  top-level resource component.

* Add support for component resource output properties.

* Add get_project() and get_stack() functions for retrieving the current
  project and stack names.

* Properly use UNKNOWN sentinels.

* Add a set_outputs() function on Resource.  This is defined by the
  code-generator and allows custom logic for output property setting.
  This is cleaner than the way we do this in Node.js, and gives us a
  way to ensure that output properties are "real" properties, complete
  with member documentation.  This also gives us a hook to perform
  name demangling, which the code-generator typically controls anyway.

* Add package dependencies to setuptools.py and requirements.txt.

sdk/nodejs/.gitignore

@@ -1,6 +1,5 @@
 /bin/
 /coverage/
 /node_modules/
-/proto/
 /custom_node/
 /runtime/native/node_dev/

sdk/nodejs/Makefile

@@ -26,7 +26,6 @@ lint::
 build::
 	go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST}
 	cd runtime/native && node-gyp configure
-	cp -R ../proto/nodejs/. proto/
 	cd runtime/native && node-gyp build
 	tsc
 	cp README.md ../../LICENSE package.json ./dist/* bin/

sdk/nodejs/proto/languages_grpc_pb.js

@@ -0,0 +1,122 @@
+// GENERATED CODE -- DO NOT EDIT!
+
+// Original file comments:
+// Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+//
+'use strict';
+var grpc = require('grpc');
+var languages_pb = require('./languages_pb.js');
+var google_protobuf_struct_pb = require('google-protobuf/google/protobuf/struct_pb.js');
+var provider_pb = require('./provider_pb.js');
+
+function serialize_pulumirpc_InvokeRequest(arg) {
+  if (!(arg instanceof provider_pb.InvokeRequest)) {
+    throw new Error('Expected argument of type pulumirpc.InvokeRequest');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_InvokeRequest(buffer_arg) {
+  return provider_pb.InvokeRequest.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_InvokeResponse(arg) {
+  if (!(arg instanceof provider_pb.InvokeResponse)) {
+    throw new Error('Expected argument of type pulumirpc.InvokeResponse');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_InvokeResponse(buffer_arg) {
+  return provider_pb.InvokeResponse.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_NewResourceRequest(arg) {
+  if (!(arg instanceof languages_pb.NewResourceRequest)) {
+    throw new Error('Expected argument of type pulumirpc.NewResourceRequest');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_NewResourceRequest(buffer_arg) {
+  return languages_pb.NewResourceRequest.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_NewResourceResponse(arg) {
+  if (!(arg instanceof languages_pb.NewResourceResponse)) {
+    throw new Error('Expected argument of type pulumirpc.NewResourceResponse');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_NewResourceResponse(buffer_arg) {
+  return languages_pb.NewResourceResponse.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_RunRequest(arg) {
+  if (!(arg instanceof languages_pb.RunRequest)) {
+    throw new Error('Expected argument of type pulumirpc.RunRequest');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_RunRequest(buffer_arg) {
+  return languages_pb.RunRequest.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_RunResponse(arg) {
+  if (!(arg instanceof languages_pb.RunResponse)) {
+    throw new Error('Expected argument of type pulumirpc.RunResponse');
+  }
+  return new Buffer(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_RunResponse(buffer_arg) {
+  return languages_pb.RunResponse.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+
+// LanguageRuntime is the interface that the planning monitor uses to drive execution of an interpreter responsible
+// for confguring and creating resource objects.
+var LanguageRuntimeService = exports.LanguageRuntimeService = {
+  run: {
+    path: '/pulumirpc.LanguageRuntime/Run',
+    requestStream: false,
+    responseStream: false,
+    requestType: languages_pb.RunRequest,
+    responseType: languages_pb.RunResponse,
+    requestSerialize: serialize_pulumirpc_RunRequest,
+    requestDeserialize: deserialize_pulumirpc_RunRequest,
+    responseSerialize: serialize_pulumirpc_RunResponse,
+    responseDeserialize: deserialize_pulumirpc_RunResponse,
+  },
+};
+
+exports.LanguageRuntimeClient = grpc.makeGenericClientConstructor(LanguageRuntimeService);
+// ResourceMonitor is the interface a source uses to talk back to the planning monitor orchestrating the execution.
+var ResourceMonitorService = exports.ResourceMonitorService = {
+  invoke: {
+    path: '/pulumirpc.ResourceMonitor/Invoke',
+    requestStream: false,
+    responseStream: false,
+    requestType: provider_pb.InvokeRequest,
+    responseType: provider_pb.InvokeResponse,
+    requestSerialize: serialize_pulumirpc_InvokeRequest,
+    requestDeserialize: deserialize_pulumirpc_InvokeRequest,
+    responseSerialize: serialize_pulumirpc_InvokeResponse,
+    responseDeserialize: deserialize_pulumirpc_InvokeResponse,
+  },
+  newResource: {
+    path: '/pulumirpc.ResourceMonitor/NewResource',
+    requestStream: false,
+    responseStream: false,
+    requestType: languages_pb.NewResourceRequest,
+    responseType: languages_pb.NewResourceResponse,
+    requestSerialize: serialize_pulumirpc_NewResourceRequest,
+    requestDeserialize: deserialize_pulumirpc_NewResourceRequest,
+    responseSerialize: serialize_pulumirpc_NewResourceResponse,
+    responseDeserialize: deserialize_pulumirpc_NewResourceResponse,
+  },
+};
+
+exports.ResourceMonitorClient = grpc.makeGenericClientConstructor(ResourceMonitorService);

sdk/proto/generate.sh

@@ -29,13 +29,13 @@ echo -e "\tGo: $GO_PULUMIRPC [$GO_PROTOFLAGS]"
 mkdir -p $GO_PULUMIRPC
 $PROTOC --go_out=$GO_PROTOFLAGS:$GO_PULUMIRPC *.proto
 
-JS_PULUMIRPC=./nodejs
+JS_PULUMIRPC=../nodejs/proto/
 JS_PROTOFLAGS="import_style=commonjs,binary"
 echo -e "\tJS: $JS_PULUMIRPC [$JS_PROTOFLAGS]"
 mkdir -p $JS_PULUMIRPC
 $PROTOC --js_out=$JS_PROTOFLAGS:$JS_PULUMIRPC --grpc_out=$JS_PULUMIRPC --plugin=protoc-gen-grpc=`which grpc_tools_node_protoc_plugin` *.proto
 
-PY_PULUMIRPC=./python
+PY_PULUMIRPC=../python/lib/pulumi/runtime/proto/
 echo -e "\tPython: $PY_PULUMIRPC"
 mkdir -p $PY_PULUMIRPC
 python -m grpc_tools.protoc -I./ --python_out=$PY_PULUMIRPC --grpc_python_out=$PY_PULUMIRPC *.proto

sdk/python/Makefile

@@ -1,6 +1,6 @@
 PROJECT_NAME     := Pulumi Python SDK
 LANGHOST_PKG     := github.com/pulumi/pulumi/sdk/python/cmd/pulumi-language-python
-VERSION          := $(shell git describe --tags --dirty 2>/dev/null)
+VERSION          := $(shell git describe --tags --dirty 2>/dev/null | sed "s/v//" | sed "s/-/+/" | sed "s/-/./g")
 
 GOMETALINTERBIN := gometalinter
 GOMETALINTER    := ${GOMETALINTERBIN} --config=../../Gometalinter.json
@@ -11,16 +11,18 @@ ensure::
 	pip install -r requirements.txt
 
 lint::
-	find . -path ./bin -prune -o -name '*.py' -print | xargs pylint -E
+	pylint -E lib/pulumi/ --ignore-patterns '.*_pb2_.*.py'
 	$(GOMETALINTER) ./cmd/pulumi-language-python/... | sort ; exit $${PIPESTATUS[0]}
 	$(GOMETALINTER) ./pkg/... | sort ; exit $${PIPESTATUS[0]}
 
 build::
+	cd ./lib/ && python setup.py clean --all 2>/dev/null
+	rm -rf ./bin/ && cp -R ./lib/. ./bin/
+	sed -i.bak "s/\$${VERSION}/$(VERSION)/g" ./bin/setup.py && rm ./bin/setup.py.bak
+	cd ./bin/ && python setup.py build
 	go install -ldflags "-X github.com/pulumi/pulumi/sdk/python/pkg/version.Version=${VERSION}" ${LANGHOST_PKG}
 
 install::
-	cp -R ./lib/. ./bin/
-	sed -i.bak "s/\$${VERSION}/$(VERSION)/g" ./bin/setup.py && rm ./bin/setup.py.bak
 	cd ./bin/ && python setup.py install --force
 	cp ./cmd/pulumi-language-python-exec "$(PULUMI_BIN)"
 	GOBIN=$(PULUMI_BIN) go install \

sdk/python/cmd/pulumi-language-python-exec

@@ -45,7 +45,11 @@ if __name__ == "__main__":
     if not args.pwd is None:
         os.chdir(args.pwd)
     try:
-        runpy.run_path(args.PROGRAM, run_name='__main__')
+        try:
+            pulumi.runtime.run_in_stack(lambda: runpy.run_path(args.PROGRAM, run_name='__main__'))
+        finally:
+            sys.stdout.flush()
+            sys.stderr.flush()
     except pulumi.RunError as e:
         sys.stderr.write(e.message)
         sys.exit(1)

sdk/python/lib/pulumi/__init__.py

@@ -10,4 +10,5 @@ __all__ = ['runtime']
 # Make all module members inside of this package available as package members.
 from config import *
 from errors import *
+from metadata import *
 from resource import *

sdk/python/lib/pulumi/config.py

@@ -5,7 +5,7 @@ The config module contains all configuration management functionality.
 """
 
 import errors
-import runtime
+from runtime.config import get_config
 
 class Config(object):
     """
@@ -27,7 +27,7 @@ class Config(object):
         """
         Returns an optional configuration value by its key, or None if it doesn't exist.
         """
-        return runtime.get_config(self.full_key(key))
+        return get_config(self.full_key(key))
 
     def get_bool(self, key):
         """

sdk/python/lib/pulumi/metadata.py

@@ -0,0 +1,15 @@
+# Copyright 2016-2018, Pulumi Corporation.  All rights reserved.
+
+from runtime import SETTINGS
+
+def get_project():
+    """
+    Returns the current project name.
+    """
+    return SETTINGS.project
+
+def get_stack():
+    """
+    Returns the current stack name.
+    """
+    return SETTINGS.stack

sdk/python/lib/pulumi/resource.py

@@ -2,7 +2,8 @@
 
 """The Resource module, containing all resource-related definitions."""
 
-from pulumi import runtime
+from runtime.resource import register_resource, register_resource_outputs
+from runtime.settings import get_root_resource
 
 class Resource(object):
     """
@@ -32,12 +33,18 @@ class Resource(object):
 
         # Default the parent if there is none.
         if opts.parent is None:
-            opts.parent = runtime.get_root_resource() # pylint: disable=assignment-from-none
+            opts.parent = get_root_resource()
 
         # Now register the resource.  If we are actually performing a deployment, this resource's properties
         # will be resolved to real values.  If we are only doing a dry-run preview, on the other hand, they will
         # resolve to special Preview sentinel values to indicate the value isn't yet available.
-        runtime.register_resource(self, t, name, custom, props, opts)
+        register_resource(self, t, name, custom, props, opts)
+
+    def set_outputs(self, outputs):
+        """
+        Sets output properties after a registration has completed.
+        """
+        # By default, do nothing.  If subclasses wish to support provider outputs, they must override this.
 
 class ResourceOptions(object):
     """
@@ -65,8 +72,18 @@ class ComponentResource(Resource):
     def __init__(self, t, name, props=None, opts=None):
         Resource.__init__(self, t, name, False, props, opts)
 
+    def register_outputs(self, outputs):
+        """
+        Register synthetic outputs that a component has initialized, usually by allocating other child
+        sub-resources and propagating their resulting property values.
+        """
+        if outputs:
+            register_resource_outputs(self, outputs)
+
 def export(name, value):
     """
     Exports a named stack output.
     """
-    # TODO
+    stack = get_root_resource()
+    if stack is not None:
+        stack.export(name, value)

sdk/python/lib/pulumi/runtime/__init__.py

@@ -8,3 +8,4 @@ The runtime implementation of the Pulumi Python SDK.
 from config import *
 from resource import *
 from settings import *
+from stack import *

sdk/python/lib/pulumi/runtime/proto/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+
+"""
+The Pulumi SDK runtime's Protobufs and gRPC stubs.  These are meant for internal use only.
+"""
+
+from analyzer_pb2 import *
+from analyzer_pb2_grpc import *
+from engine_pb2 import *
+from engine_pb2_grpc import *
+from language_pb2 import *
+from language_pb2_grpc import *
+from plugin_pb2 import *
+from plugin_pb2_grpc import *
+from provider_pb2 import *
+from provider_pb2_grpc import *
+from resource_pb2 import *
+from resource_pb2_grpc import *

sdk/python/lib/pulumi/runtime/resource.py

@@ -4,14 +4,105 @@
 Resource-related runtime functions.  These are not designed for external use.
 """
 
-def register_resource(res, typ, name, custom, props, opts): # pylint: disable=unused-argument
+from ..errors import RunError
+from google.protobuf import struct_pb2
+from proto import resource_pb2
+from settings import get_monitor
+import six
+import sys
+
+def register_resource(res, typ, name, custom, props, opts):
     """
     Registers a new resource object with a given type and name.  This call is synchronous while the resource is
     created and All properties will be initialized to real property values once it completes.
     """
 
-def get_root_resource():
+    # Serialize all properties.  This just translates known types into the gRPC marshalable equivalents.
+    objprops = serialize_resource_props(props)
+
+    # Ensure we have flushed all stdout/stderr, in case the RPC fails.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # Now perform the resource registration.  This is synchronous and will return only after the operation completes.
+    # TODO(joe): asynchronous registration to support parallelism.
+    monitor = get_monitor()
+    resp = monitor.RegisterResource(resource_pb2.RegisterResourceRequest(
+        type=typ,
+        name=name,
+        parent=opts.parent.urn if opts and opts.parent else None,
+        custom=custom,
+        object=objprops,
+        protect=opts.protect if opts else None))
+
+    # Now copy the URN and ID properties back onto the resource object.
+    res.urn = resp.urn
+    if custom:
+        if resp.id is None or resp.id == "":
+            res.id = UNKNOWN
+        else:
+            res.id = resp.id
+
+    # Now let the class itself decide how to accept output properties, if desired.
+    if resp.object:
+        outs = dict()
+        for k, v in resp.object.items():
+            outs[k] = v
+        res.set_outputs(outs)
+
+def register_resource_outputs(res, outputs):
     """
-    Returns the implicit root stack resource for all resources created in this program.
+    Registers custom resource output properties.  This call is serial and blocks until the registration completes.
     """
-    return None
+
+    # Serialize all properties.  This just translates known types into the gRPC marshalable equivalents.
+    objouts = serialize_resource_props(outputs)
+
+    # Ensure we have flushed all stdout/stderr, in case the RPC fails.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # Now perform the output registration.  This is synchronous and will return only after the operation completes.
+    # TODO(joe): asynchronous registration to support parallelism.
+    monitor = get_monitor()
+    monitor.RegisterResourceOutputs(resource_pb2.RegisterResourceOutputsRequest(
+        urn=res.urn,
+        outputs=objouts))
+
+def serialize_resource_props(props):
+    """
+    Serializes resource properties so that they are ready for marshaling to the gRPC endpoint.
+    """
+    struct = struct_pb2.Struct()
+    for k, v in props.items():
+        struct[k] = serialize_resource_value(v) # pylint: disable=unsupported-assignment-operation
+    return struct
+
+from ..resource import CustomResource
+
+UNKNOWN = "04da6b54-80e4-46f7-96ec-b56ff0331ba9"
+"""If a value is None, we serialize as UNKNOWN, which tells the engine that it may be computed later."""
+
+def serialize_resource_value(value):
+    """
+    Seralizes a resource property value so that it's ready for marshaling to the gRPC endpoint.
+    """
+    if isinstance(value, CustomResource):
+        # Resource objects aren't serializable.  Instead, serialize them as references to their IDs.
+        return serialize_resource_value(value.id)
+    elif isinstance(value, dict):
+        # Deeply serialize dictionaries.
+        d = dict()
+        for k, v in value.items():
+            d[k] = serialize_resource_value(v)
+        return d
+    elif isinstance(value, list):
+        # Deeply serialize lists.
+        a = []
+        for e in value:
+            a.append(serialize_resource_value(e))
+        return a
+    else:
+        # All other values are directly serializable.
+        # TODO(joe): eventually, we want to think about Output, Properties, and so on.
+        return value

sdk/python/lib/pulumi/runtime/settings.py

@@ -4,18 +4,27 @@
 Runtime settings and configuration.
 """
 
+import grpc
+from proto import engine_pb2_grpc, resource_pb2_grpc
+from ..errors import RunError
+
 class Settings(object):
     """
     A bag of properties for configuring the Pulumi Python language runtime.
     """
     def __init__(self, monitor=None, engine=None, project=None, stack=None, parallel=None, dry_run=None):
-        self.monitor = monitor
-        self.engine = engine
+        # Save the metadata information.
         self.project = project
         self.stack = stack
         self.parallel = parallel
         self.dry_run = dry_run
 
+        # Actually connect to the monitor/engine over gRPC.
+        if monitor:
+            self.monitor = resource_pb2_grpc.ResourceMonitorStub(grpc.insecure_channel(monitor))
+        if engine:
+            self.engine = engine_pb2_grpc.EngineStub(grpc.insecure_channel(engine))
+
 # default to "empty" settings.
 SETTINGS = Settings()
 
@@ -27,3 +36,28 @@ def configure(settings):
         raise TypeError('Settings is expected to be non-None and of type Settings')
     global SETTINGS # pylint: disable=global-statement
     SETTINGS = settings
+
+def get_monitor():
+    """
+    Returns the current resource monitoring service client for RPC communications.
+    """
+    monitor = SETTINGS.monitor
+    if not monitor:
+        raise RunError('Pulumi program not connected to the engine -- are you running with the `pulumi` CLI?')
+    return monitor
+
+ROOT = None
+
+def get_root_resource():
+    """
+    Returns the implicit root stack resource for all resources created in this program.
+    """
+    global ROOT
+    return ROOT
+
+def set_root_resource(root):
+    """
+    Sets the current root stack resource for all resources subsequently to be created in this program.
+    """
+    global ROOT
+    ROOT = root

sdk/python/lib/pulumi/runtime/stack.py

@@ -0,0 +1,44 @@
+# Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+
+"""
+Support for automatic stack components.
+"""
+
+from ..resource import ComponentResource
+from settings import get_root_resource, set_root_resource, SETTINGS
+
+def run_in_stack(func):
+    """
+    Run the given function inside of a new stack resource.  This ensures that any stack export calls will end
+    up as output properties on the resulting stack component in the checkpoint file.  This is meant for internal
+    runtime use only and is used by the Python SDK entrypoint program.
+    """
+    Stack(func)
+
+class Stack(ComponentResource):
+    """
+    A synthetic stack component that automatically parents resources as the program runs.
+    """
+    def __init__(self, func):
+        # Ensure we don't already have a stack registered.
+        if get_root_resource() is not None:
+            raise Exception('Only one root Pulumi Stack may be active at once')
+
+        # Now invoke the registration to begin creating this resource.
+        name = '%s-%s' % (SETTINGS.project, SETTINGS.stack)
+        super(Stack, self).__init__('pulumi:pulumi:Stack', name, None, None)
+
+        # Invoke the function while this stack is active and then register its outputs.
+        self.outputs = dict()
+        set_root_resource(self)
+        try:
+            func()
+        finally:
+            self.register_outputs(self.outputs)
+            # Intentionally leave this resource installed in case subsequent async work uses it.
+
+    def export(self, name, value):
+        """
+        Export a stack output with a given name and value.
+        """
+        self.outputs[name] = value

sdk/python/lib/setup.py

@@ -2,11 +2,16 @@
 
 """The Pulumi Python SDK."""
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(name='pulumi',
       version='${VERSION}',
       description='Pulumi\'s Python SDK',
       url='https://github.com/pulumi/pulumi',
-      packages=['pulumi', 'pulumi.runtime'],
+      packages=find_packages(),
+      install_requires=[
+          'google==2.0.1',
+          'grpcio==1.9.1',
+          'six==1.11.0'
+      ],
       zip_safe=False)

sdk/python/requirements.txt

@@ -1 +1,4 @@
-pylint==1.5.5
+google==2.0.1
+grpcio==1.9.1
+pylint==1.6.0
+six==1.11.0