maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Invoke node directly from the language host

Browse source 
Instead of using a shell script to jump from the language host into
node, just invoke node directly. This makes our start-up path a little
simpler to understand and indirectly fixes pulumi/home#156, where we
would fail on Windows if the `-exec` script was in a folder that had
spaces in it (due to a subtle interaction between how go launches cmd
files and how cmd.exe parses arguments).
This commit is contained in:
Matt Ellis 2018-04-30 16:10:01 -07:00
parent c442ae70ca
commit 409477b951
9 changed files with 66 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
build.proj
View file

@ -22,9 +22,9 @@
<Target Name="EnsurePrebuilt">
<MakeDir Directories="$(NodeJSSdkDirectory)\prebuilt" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/nativeruntime.node $(NodeJSSdkDirectory)\prebuilt\nativeruntime.node" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/nativeruntime-v0.11.0.node $(NodeJSSdkDirectory)\prebuilt\nativeruntime-v0.11.0.node" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/pulumi-language-nodejs-node.exe $(NodeJSSdkDirectory)\prebuilt\pulumi-language-nodejs-node.exe" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/nativeruntime.node &quot;$(NodeJSSdkDirectory)\prebuilt\nativeruntime.node&quot;" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/nativeruntime-v0.11.0.node &quot;$(NodeJSSdkDirectory)\prebuilt\nativeruntime-v0.11.0.node&quot;" />
<Exec Command="aws s3 cp s3://eng.pulumi.com/nativeruntime/windows/pulumi-language-nodejs-node.exe &quot;$(NodeJSSdkDirectory)\prebuilt\pulumi-language-nodejs-node.exe&quot;" />
</Target>
<Target Name="TypeScriptCompileNodeSdk">
@ -33,8 +33,8 @@
</Exec>
<Exec Command="yarn run tsc" WorkingDirectory="$(NodeJSSdkDirectory)" />
<Copy SourceFiles="$(NodeJSSdkDirectory)\package.json" DestinationFiles="$(NodeJSSdkDirectory)\bin\package.json" />
<Exec Command="node $(RepoRootDirectory)\scripts\reversion.js $(NodeJSSdkDirectory)\bin\package.json $(Version)" />
<Exec Command="node $(RepoRootDirectory)\scripts\reversion.js $(NodeJSSdkDirectory)\bin\version.js $(Version)" />
<Exec Command="node &quot;$(RepoRootDirectory)\scripts\reversion.js&quot; &quot;$(NodeJSSdkDirectory)\bin\package.json&quot; $(Version)" />
<Exec Command="node &quot;$(RepoRootDirectory)\scripts\reversion.js&quot; &quot;$(NodeJSSdkDirectory)\bin\version.js&quot; $(Version)" />
</Target>
<Target Name="GoCompileNodeSdk">
@ -75,7 +75,6 @@
<Target Name="BinPlaceNodeSdk"
DependsOnTargets="BinPlaceNodeSdkProtos;BinPlaceNodeSdkTestData;YarnLinkSdk">
<Copy SourceFiles="$(NodeJSSdkDirectory)\dist\pulumi-language-nodejs-exec.cmd" DestinationFolder="$(PulumiBin)" />
<Copy SourceFiles="$(NodeJSSdkDirectory)\dist\pulumi-resource-pulumi-nodejs.cmd" DestinationFolder="$(PulumiBin)" />
<MakeDir Directories="$(PulumiBin)\v6.10.2" />
<MakeDir Directories="$(PulumiBin)\custom_node" />
@ -106,11 +105,6 @@
</Target>
<Target Name="IntegrationTest">
<Exec Command="where pulumi-language-nodejs-exec.cmd"
IgnoreExitCode="true">
<Output TaskParameter="ExitCode" PropertyName="WhereLangHostExecExitCode" />
</Exec>
<Exec Command="where pulumi-language-nodejs"
IgnoreExitCode="true">
<Output TaskParameter="ExitCode" PropertyName="WhereLangHostExitCode" />
@ -118,11 +112,11 @@
<Exec Command="where pulumi-resource-pulumi-nodejs.cmd"
IgnoreExitCode="true">
<Output TaskParameter="ExitCode" PropertyName="WhereLangHostExitCode" />
<Output TaskParameter="ExitCode" PropertyName="WhereDynamicProviderExitCode" />
</Exec>
<Error Text="Please add &quot;$(PulumiRoot)\bin&quot; to your path before running integration tests."
Condition="$(WhereLangHostExitCode) != 0 Or $(WhereLangHostExecExitCode) != 0"/>
Condition="$(WhereLangHostExitCode) != 0 Or $(WhereDynamicProviderExitCode) != 0"/>
<!-- Ignore the exit code (but retain it) so we can kill all the lingering node processes even when go test
fails. Otherwise, the AppVeyor job would hang until it reached the timeout -->

1
scripts/make_release.ps1
View file

@ -32,7 +32,6 @@ RunGoBuild "github.com/pulumi/pulumi"
RunGoBuild "github.com/pulumi/pulumi/sdk/nodejs/cmd/pulumi-language-nodejs"
CopyPackage "$Root\sdk\nodejs\bin" "pulumi"
Copy-Item "$Root\sdk\nodejs\dist\pulumi-language-nodejs-exec.cmd" "$PublishDir\bin"
Copy-Item "$Root\sdk\nodejs\dist\pulumi-resource-pulumi-nodejs.cmd" "$PublishDir\bin"
New-Item -ItemType Directory -Path "$PublishDir\bin\v6.10.2" | Out-Null

1
scripts/make_release.sh
View file

@ -49,7 +49,6 @@ run_go_build "${ROOT}/sdk/nodejs/cmd/pulumi-language-nodejs"
run_go_build "${ROOT}/sdk/python/cmd/pulumi-language-python"
# Copy over the language and dynamic resource providers.
cp ${ROOT}/sdk/nodejs/dist/pulumi-language-nodejs-exec ${PUBDIR}/bin/
cp ${ROOT}/sdk/nodejs/dist/pulumi-resource-pulumi-nodejs ${PUBDIR}/bin/
cp ${ROOT}/sdk/python/cmd/pulumi-language-python-exec ${PUBDIR}/bin/

1
sdk/nodejs/Makefile
View file

@ -36,7 +36,6 @@ build::
install::
GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/version.Version=${VERSION}" ${LANGUAGE_HOST}
cp dist/pulumi-language-nodejs-exec "$(PULUMI_BIN)"
cp dist/pulumi-resource-pulumi-nodejs "$(PULUMI_BIN)"
mkdir -p "$(PULUMI_BIN)/v6.10.2"
cp ./prebuilt/*.node "$(PULUMI_BIN)/v6.10.2/"

86
sdk/nodejs/cmd/pulumi-language-nodejs/main.go
View file

@ -39,9 +39,9 @@ import (
)
const (
// By convention, the executor is the name of the current program
// (pulumi-language-nodejs) plus this suffix.
nodeExecSuffix = "-exec" // the exec shim for Pulumi to run Node programs.
// The path to the "run" program which will spawn the rest of the language host. This may be overriden with
// PULUMI_LANGUAGE_NODEJS_RUN_PATH, which we do in some testing cases.
defaultRunPath = "./node_modules/@pulumi/pulumi/cmd/run"
// The runtime expects the config object to be saved to this environment variable.
pulumiConfigVar = "PULUMI_CONFIG"
@ -52,40 +52,27 @@ const (
// endpoint.
func main() {
var tracing string
var givenExecutor string
flag.StringVar(&tracing, "tracing", "",
"Emit tracing to a Zipkin-compatible tracing endpoint")
// You can use the below flag to request that the language host load
// a specific executor instead of probing the PATH. This is used specifically
// in run.spec.ts to work around some unfortunate Node module loading behavior.
flag.StringVar(&givenExecutor, "use-executor", "",
"Use the given program as the executor instead of looking for one on PATH")
flag.Parse()
args := flag.Args()
cmdutil.InitLogging(false, 0, false)
cmdutil.InitTracing(os.Args[0], tracing)
var nodeExec string
if givenExecutor == "" {
// The -exec binary is the same name as the current language host, except that we must trim off
// the file extension (if any) and then append -exec to it.
bin := os.Args[0]
if ext := filepath.Ext(bin); ext != "" {
bin = bin[:len(bin)-len(ext)]
}
bin += nodeExecSuffix
pathExec, err := exec.LookPath(bin)
if err != nil {
err = errors.Wrapf(err, "could not find `%s` on the $PATH", bin)
cmdutil.Exit(err)
}
glog.V(3).Infof("language host identified executor from path: `%s`", pathExec)
nodeExec = pathExec
} else {
glog.V(3).Infof("language host asked to use specific executor: `%s`", givenExecutor)
nodeExec = givenExecutor
nodePath, err := exec.LookPath("node")
if err != nil {
cmdutil.Exit(errors.Wrapf(err, "could not find node on the $PATH"))
}
runPath := os.Getenv("PULUMI_LANGUAGE_NODEJS_RUN_PATH")
if runPath == "" {
runPath = defaultRunPath
}
if _, err = os.Stat(runPath); err != nil {
cmdutil.ExitError(
"It looks like the Pulumi SDK has not been installed. Have you run npm install or yarn install?")
}
// Optionally pluck out the engine so we can do logging, etc.
@ -97,7 +84,7 @@ func main() {
// Fire up a gRPC server, letting the kernel choose a free port.
port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{
func(srv *grpc.Server) error {
host := newLanguageHost(nodeExec, engineAddress, tracing)
host := newLanguageHost(nodePath, runPath, engineAddress, tracing)
pulumirpc.RegisterLanguageRuntimeServer(srv, host)
return nil
},
@ -118,14 +105,16 @@ func main() {
// nodeLanguageHost implements the LanguageRuntimeServer interface
// for use as an API endpoint.
type nodeLanguageHost struct {
exec string
nodeBin string
runPath string
engineAddress string
tracing string
}
func newLanguageHost(exec, engineAddress, tracing string) pulumirpc.LanguageRuntimeServer {
func newLanguageHost(nodePath, runPath, engineAddress, tracing string) pulumirpc.LanguageRuntimeServer {
return &nodeLanguageHost{
exec: exec,
nodeBin: nodePath,
runPath: runPath,
engineAddress: engineAddress,
tracing: tracing,
}
@ -263,17 +252,40 @@ func (host *nodeLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest
return nil, err
}
ourCmd, err := os.Executable()
if err != nil {
err = errors.Wrap(err, "failed to find our working directory")
return nil, err
}
// Older versions of the pulumi runtime used a custom node module (which only worked on node 6.10.X) to support
// closure serialization. While we no longer use this, we continue to ship this module with the language host in
// the SDK, so we can deploy programs using older versions of the Pulumi framework. So, for now, let's add this
// folder with our native modules to the NODE_PATH so Node can find it.
//
// TODO(ellismg)[pulumi/pulumi#1298]: Remove this block of code when we no longer need to support older
// @pulumi/pulumi versions.
env := os.Environ()
existingNodePath := os.Getenv("NODE_PATH")
if existingNodePath != "" {
env = append(env, fmt.Sprintf("NODE_PATH=%s/v6.10.2:%s", filepath.Dir(ourCmd), existingNodePath))
} else {
env = append(env, "NODE_PATH="+filepath.Dir(ourCmd)+"/v6.10.2")
}
env = append(env, pulumiConfigVar+"="+string(config))
if glog.V(5) {
commandStr := strings.Join(args, " ")
glog.V(5).Infoln("Language host launching process: ", host.exec, commandStr)
glog.V(5).Infoln("Language host launching process: ", host.nodeBin, commandStr)
}
// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
var errResult string
cmd := exec.Command(host.exec, args...) // nolint: gas, intentionally running dynamic program name.
cmd := exec.Command(host.nodeBin, args...) // nolint: gas, intentionally running dynamic program name.
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = append(os.Environ(), pulumiConfigVar+"="+string(config))
cmd.Env = env
if err := cmd.Run(); err != nil {
if exiterr, ok := err.(*exec.ExitError); ok {
// If the program ran, but exited with a non-zero error code. This will happen often, since user
@ -299,7 +311,7 @@ func (host *nodeLanguageHost) Run(ctx context.Context, req *pulumirpc.RunRequest
// by enumerating all of the optional and non-optional arguments present
// in a RunRequest.
func (host *nodeLanguageHost) constructArguments(req *pulumirpc.RunRequest) []string {
var args []string
args := []string{host.runPath}
maybeAppendArg := func(k, v string) {
if v != "" {
args = append(args, "--"+k, v)

12
sdk/nodejs/dist/pulumi-language-nodejs-exec vendored
View file

@ -1,12 +0,0 @@
#!/bin/sh
# we exploit the fact that the cwd when `pulumi-language-nodejs-exec` is
# run is the root of the node program we want to run and use a relative
# path here.
export NODE_PATH="$NODE_PATH:`dirname $0`/v6.10.2"
PULUMI_RUN=./node_modules/@pulumi/pulumi/cmd/run
if [ ! -e $PULUMI_RUN ]; then
echo "It looks like the Pulumi SDK has not been installed. Have you run npm install or yarn install?"
exit 1
fi
node $PULUMI_RUN $@

2
sdk/nodejs/dist/pulumi-language-nodejs-exec-test vendored
View file

@ -1,2 +0,0 @@
#!/bin/sh
node ./bin/cmd/run $@

10
sdk/nodejs/dist/pulumi-language-nodejs-exec.cmd vendored
View file

@ -1,10 +0,0 @@
@echo off
set NODE_PATH=%NODE_PATH%;%~dp0\v6.10.2
set PULUMI_RUN=./node_modules/@pulumi/pulumi/cmd/run
if not exist %PULUMI_RUN% (
echo It looks like the Pulumi SDK has not been installed. Have you run npm install or yarn install?
exit /b 1
)
node %PULUMI_RUN% %*

35
sdk/nodejs/tests/runtime/langhost/run.spec.ts
View file

@ -544,32 +544,17 @@ function createMockResourceMonitor(
function serveLanguageHostProcess(): { proc: childProcess.ChildProcess, addr: Promise<string> } {
// A quick note about this:
//
// Normally, pulumi-language-nodejs probes the path in order to
// find the nodejs executor, pulumi-language-nodejs-exec. This works
// great in all scenarios other than testing within this file. If the executor
// that it founds resides in the Pulumi install dir (which it will, if these tests
// are being executed by `make`), then Node will execute it by resolving our relative
// path requires to the Pulumi install directory. However, the programs being evaluated
// by the language host are using the current directory to resolve relative path requires.
// Normally, `pulumi-language-nodejs` launches `./node-modules/@pulumi/pulumi/cmd/run` which is responsible
// for setting up some state and then running the actual user program. However, in this case, we don't
// have a folder structure like the above because we are seting the package as we've built it, not it installed
// in another application.
//
// Normally, this isn't a problem - ostensibly the stuff in the Pulumi install directory
// is the same as the stuff that we are currently testing. However, the "settings.ts" module
// contains some global state that is expected to be shared between the language host stub
// that is connecting to our RPC endpoints (run/index.ts) and the Pulumi program being
// evaluated. Node, when resolving the require, will pick different modules to load depending
// on which context the require occured; if it happened in run/index.ts, it'll load
// from Pulumi install directory, while requires coming from anywhere else will load
// from the source directory. Because these are two different files, Node instantiates two
// separate module objects and the state that we are expecting to share is not actually shared,
// ultimately resulting in extremely wacky errors.
//
// In order to work around this problem, the langhost is explicitly instructed
// (through --use-executor) to use a specific executor which will load modules from
// the source directory and not the install directory.
const proc = childProcess.spawn("pulumi-language-nodejs", [
"--use-executor",
path.join(__filename, "..", "..", "..", "..", "pulumi-language-nodejs-exec-test"),
]);
// `pulumi-language-nodejs` allows us to set `PULUMI_LANGUAGE_NODEJS_RUN_PATH` in the environment, and when
// set, it will use that path instead of the default value. For our tests here, we set it and point at the
// just built version of run.
process.env.PULUMI_LANGUAGE_NODEJS_RUN_PATH = "./bin/cmd/run";
const proc = childProcess.spawn("pulumi-language-nodejs");
// Hook the first line so we can parse the address. Then we hook the rest to print for debugging purposes, and
// hand back the resulting process object plus the address we plucked out.
let addrResolve: ((addr: string) => void) | undefined;