WIP

.gitignore

@@ -14,6 +14,10 @@ coverage.cov
 **/.idea/
 *.iml
 
+# Ignoring the task folder where it calculates the checksums for the files to understand if it
+# needs to do work
+.task/
+
 # VSCode creates this binary when running tests in the debugger
 **/debug.test
 

Taskfile.yml

@@ -0,0 +1,111 @@
+version: '3'
+
+vars:
+  PULUMI_ROOT: /opt/pulumi
+  PULUMI_BIN: "{{ .PULUMI_ROOT }}/bin"
+  PULUMI_NODE_MODULES: "{{ .PULUMI_ROOT }}/node_modules"
+  PULUMI_NUGET: "{{ .PULUMI_ROOT }}/nuget"
+  PULUMI_LOCAL_NUGET: "{{ .PULUMI_NUGET }}"
+  TESTPARALLELISM: 10
+
+tasks:
+  __default:
+    desc: A dev wrapper around ensure, lint, build + install
+    cmds:
+      - task: ensure
+      - task: lint
+      - task: build
+      - task: install
+      - task: unit-test
+
+  ensure:
+    cmds:
+      - task: pkg:ensure
+      - task: sdk:ensure
+      - task: tests:ensure
+      - task: dotnet:ensure
+      - task: go:ensure
+      - task: nodejs:ensure
+      - task: python:ensure
+
+  lint:
+    desc: Lint the CLI, Language SDKs and the tests folder
+    cmds:
+      - task: pkg:lint
+      - task: sdk:lint
+      - task: tests:lint
+      - task: dotnet:lint
+      - task: go:lint
+      - task: nodejs:lint
+      - task: python:lint
+
+  build-proto:
+    desc: Build the protos
+    cmds:
+      - task: proto:build-proto
+
+  build:
+    desc: Build the CLI + SDKs - this does not install the binaries
+    deps: [build-proto]
+    cmds:
+      - task: pkg:build
+      - task: sdk:build
+      - task: tests:build
+      - task: dotnet:build
+      - task: go:build
+      - task: nodejs:build
+      - task: python:build
+
+  install:
+    desc: Install the CLI + Language host binaries
+    cmds:
+      - task: pkg:install
+      - task: sdk:install
+      - task: tests:install
+      - task: dotnet:install
+      - task: go:install
+      - task: nodejs:install
+      - task: python:install
+
+  unit-test:
+    desc: Run unit tests of SDKs + CLI - these unit tests include some Automation API tests which are full integration tests
+    cmds:
+      - task: pkg:test
+      - task: dotnet:test
+      - task: go:test
+      - task: python:test
+      - task: nodejs:test
+
+  integration-tests:
+    desc: Run integration tests in tests folder - this runs all integration tests but each can be called individually
+    cmds:
+      - task tests:dotnet
+      - task tests:go
+      - task tests:nodejs
+      - task tests:python
+
+includes:
+  pkg:
+    taskfile: ./pkg/Taskfile.yml
+    dir: ./pkg
+  sdk:
+    taskfile: ./sdk/Taskfile.yml
+    dir: ./sdk
+  tests:
+    taskfile: ./tests/Taskfile.yml
+    dir: ./tests
+  dotnet:
+    taskfile: ./sdk/dotnet/Taskfile.yml
+    dir: ./sdk/dotnet
+  go:
+    taskfile: ./sdk/go/Taskfile.yml
+    dir: ./sdk/go
+  nodejs:
+    taskfile: ./sdk/nodejs/Taskfile.yml
+    dir: ./sdk/nodejs
+  python:
+    taskfile: ./sdk/python/Taskfile.yml
+    dir: ./sdk/python
+  proto:
+    taskfile: ./sdk/proto/Taskfile.yml
+    dir: ./sdk/proto

pkg/Taskfile.yml

@@ -0,0 +1,62 @@
+version: '3'
+
+vars:
+  CLI_PROJECT: "github.com/pulumi/pulumi/pkg/v3/cmd/pulumi"
+  CLI_VERSION:
+    sh: cd ../ && pulumictl get version && cd pkg
+
+tasks:
+  ensure:
+    cmds:
+      - go mod download
+    status:
+      - go mod verify
+
+  lint:
+    cmds:
+      - golangci-lint run -c ../.golangci.yml --timeout 5m
+
+  generate:
+    cmds:
+      - go generate ./codegen/docs/gen.go
+    sources:
+      - ./codegen/docs/gen.go
+    generates:
+      - ./codegen/docs/packaged.go
+    method: checksum
+
+  build:
+    deps: [generate]
+    cmds:
+      - go build -o ../bin/pulumi -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version={{.CLI_VERSION}}" {{.CLI_PROJECT}}
+    sources:
+      - ./cmd/pulumi/*.go
+    generates:
+      - ../bin/pulumi
+    method: checksum
+
+  build-debug:
+    deps: [ generate ]
+    cmds:
+      - go build -o ../bin/pulumi -gcflags="all=-N -l" -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version={{.CLI_VERSION}}" {{.CLI_PROJECT}}
+    sources:
+      - ./cmd/pulumi/*.go
+    generates:
+      - ../bin/pulumi
+    method: checksum
+
+  install:
+    deps: [build]
+    cmds:
+      - GOBIN={{.PULUMI_BIN}} go install -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version={{.CLI_VERSION}}" {{.CLI_PROJECT}}
+    sources:
+      - ./cmd/pulumi/*.go
+    generates:
+      - "{{.PULUMI_BIN}}/pulumi"
+    method: checksum
+
+  test:
+    deps: [ build ]
+    cmds:
+      - cmd: go test -count=1 -cover -timeout 1h -tags=all -parallel {{ .TESTPARALLELISM }} ./...
+        ignore_error: true

sdk/Taskfile.yml

@@ -0,0 +1,20 @@
+version: '3'
+
+tasks:
+  ensure:
+    cmds:
+      - go mod download
+    status:
+      - go mod verify
+
+  lint:
+    cmds:
+      - golangci-lint run -c ../.golangci.yml --timeout 5m
+
+  build:
+    cmds:
+      - ""
+
+  install:
+    cmds:
+      - ""

sdk/dotnet/Taskfile.yml

@@ -0,0 +1,67 @@
+version: '3'
+
+vars:
+  DOTNET_PROJECT: "github.com/pulumi/pulumi/sdk/v3/dotnet/cmd/pulumi-language-dotnet"
+  DOTNET_VERSION:
+    sh: cd ../../ && pulumictl get version --language dotnet && cd sdk/dotnet
+
+tasks:
+  ensure:
+    cmds:
+      - ""
+
+  lint:
+    cmds:
+      - ""
+
+  build-binary:
+    cmds:
+      - go build -o ../../bin/pulumi-language-dotnet -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.DOTNET_VERSION}}" {{.DOTNET_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-dotnet/*.go
+    generates:
+      - ../../bin/pulumi-language-dotnet
+    method: checksum
+
+  build-sln:
+    cmds:
+      - "dotnet build dotnet.sln /p:Version={{.DOTNET_VERSION}}"
+    sources:
+      - ./dotnet.sln
+    generates:
+      - ./Pulumi/bin/Debug/*.nupkg
+      - ./Pulumi.Automation/bin/Debug/*.nupkg
+      - ./Pulumi.FSharp/bin/Debug/*.nupkg
+    method: checksum
+
+  build:
+    cmds:
+      - task: build-sln
+      - task: build-binary
+
+  install-plugin:
+    cmds:
+      - rm -f {{.PULUMI_NUGET}}/*.nupkg
+      - find . -name '*{{.DOTNET_VERSION}}*.nupkg' -exec cp -p {} {{.PULUMI_NUGET}} \;
+
+  install:
+    cmds:
+      - task: build
+      - task: install-plugin
+      - GOBIN={{.PULUMI_BIN}} go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.DOTNET_VERSION}}" {{.DOTNET_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-dotnet/*.go
+    generates:
+      - "{{.PULUMI_BIN}}/pulumi-language-dotnet"
+    method: checksum
+
+  dotnet-test:
+    cmds:
+      - dotnet test /p:Version={{.DOTNET_VERSION}}
+
+  test:
+    cmds:
+      - task: install
+      - task: dotnet-test
+      - cmd: go test -count=1 -cover -timeout 1h -tags=all -parallel {{.TESTPARALLELISM}} ./...
+        ignore_error: true

sdk/go/Taskfile.yml

@@ -0,0 +1,56 @@
+version: '3'
+
+vars:
+  GO_PROJECT: "github.com/pulumi/pulumi/sdk/v3/python/cmd/pulumi-language-python"
+  GO_VERSION:
+    sh: cd ../../ && pulumictl get version && cd sdk/python
+
+tasks:
+  ensure:
+    cmds:
+      - ""
+
+  lint:
+    cmds:
+      - ""
+
+  generate:
+    cmds:
+      - go generate ./pulumi/...
+    sources:
+      - ./pulumi/generate.go
+    generates:
+      - ./pulumi/types_builtins.go
+      - ./pulumi/types_builtins_test.go
+      - ./pulumi/config/config.go
+      - ./pulumi/config/get.go
+      - ./pulumi/config/require.go
+      - ./pulumi/config/try.go
+    method: checksum
+
+  build:
+    deps: [ generate ]
+    cmds:
+      - go build -o ../../bin/pulumi-language-go -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.GO_VERSION}}" {{.GO_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-go/*.go
+    generates:
+      - ../../bin/pulumi-language-go
+    method: checksum
+
+  install:
+    deps: [ build ]
+    cmds:
+      - GOBIN={{.PULUMI_BIN}} go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.GO_VERSION}}" {{.GO_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-go/*.go
+    generates:
+      - "{{.PULUMI_BIN}}/pulumi-language-go"
+    method: checksum
+
+  test:
+    deps:
+      - task :install
+    cmds:
+      - cmd: go test -count=1 -cover -timeout 1h -parallel {{.TESTPARALLELISM}} ./...
+        ignore_error: true

sdk/nodejs/Taskfile.yml

@@ -0,0 +1,82 @@
+version: '3'
+
+vars:
+  NODEJS_PROJECT: "github.com/pulumi/pulumi/sdk/v3/nodejs/cmd/pulumi-language-nodejs"
+  NODE_MODULE_NAME: "@pulumi/pulumi"
+  NODEJS_VERSION:
+    sh: cd ../../ && pulumictl get version --language javascript && cd sdk/nodejs
+
+tasks:
+  ensure:
+    cmds:
+      - yarn install
+    status:
+      - yarn install --check-files
+
+  lint:
+    cmds:
+      - ./node_modules/.bin/tslint -c tslint.json -p tsconfig.json
+
+  build-package:
+    cmds:
+      - ./node_modules/.bin/tsc
+      - cp tests/runtime/jsClosureCases_8.js bin/tests/runtime
+      - cp tests/runtime/jsClosureCases_10_4.js bin/tests/runtime
+      - cp -R tests/automation/data/. bin/tests/automation/data/
+      - cp README.md ../../LICENSE package.json ./dist/* bin/
+      - node ../../scripts/reversion.js bin/package.json {{.NODEJS_VERSION}}
+      - node ../../scripts/reversion.js bin/version.js {{.NODEJS_VERSION}}
+      - cp -R proto/. bin/proto/
+      - mkdir -p bin/tests/runtime/langhost/cases/
+      - find tests/runtime/langhost/cases/* -type d -exec cp -R {} bin/tests/runtime/langhost/cases/ \;
+
+  build:
+    deps: [build-package]
+    cmds:
+      - go build -o ../../bin/pulumi-language-nodejs -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.NODEJS_VERSION}}" {{.NODEJS_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-nodejs/*.go
+    generates:
+      - ../../bin/pulumi-language-nodejs
+    method: checksum
+
+  install-plugin:
+    cmds:
+      - cp dist/pulumi-resource-pulumi-nodejs "{{.PULUMI_BIN}}"
+      - cp dist/pulumi-analyzer-policy "{{.PULUMI_BIN}}"
+
+  install-pkg:
+    cmds:
+      - mkdir -p "{{.PULUMI_NODE_MODULES}}/{{.NODE_MODULE_NAME}}"
+      - cp -r bin/. "{{.PULUMI_NODE_MODULES}}/{{.NODE_MODULE_NAME}}"
+      - cp yarn.lock "{{.PULUMI_NODE_MODULES}}/{{.NODE_MODULE_NAME}}"
+      - rm -rf "{{.PULUMI_NODE_MODULES}}/{{.NODE_MODULE_NAME}}/node_modules"
+      - cd "{{.PULUMI_NODE_MODULES}}/{{.NODE_MODULE_NAME}}" && yarn install --prefer-offline --production && (yarn unlink > /dev/null 2>&1 || true) && yarn link
+
+  install:
+    cmds:
+      - task: build
+      - task: install-plugin
+      - task: install-pkg
+      - GOBIN={{.PULUMI_BIN}} go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.NODEJS_VERSION}}" {{.NODEJS_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-nodejs/*.go
+    generates:
+      - "{{.PULUMI_BIN}}/pulumi-language-nodejs"
+    method: checksum
+
+  yarn-test:
+    cmds:
+      - ./node_modules/.bin/istanbul test --print none _mocha -- --timeout 120000 'bin/tests/**/*.spec.js'
+      - ./node_modules/.bin/istanbul report text-summary
+      - ./node_modules/.bin/istanbul report text
+      - ./node_modules/.bin/istanbul test --print none _mocha -- 'bin/tests_with_mocks/**/*.spec.js'
+      - pushd tests/sxs_ts_3.6 && yarn ; tsc ; popd
+      - pushd tests/sxs_ts_latest && yarn ; tsc ; popd
+
+  test:
+    cmds:
+      - task: install
+      - task: yarn-test
+      - cmd: go test -count=1 -cover -timeout 1h -tags=all -parallel {{ .TESTPARALLELISM }} ./...
+        ignore_error: true

sdk/proto/Taskfile.yml

@@ -0,0 +1,11 @@
+version: '3'
+
+tasks:
+  build-proto:
+    cmds:
+      - ./generate.sh
+    sources:
+      - ./*.proto
+    generates:
+      - ./go/*.pb.go
+    method: checksum

sdk/python/Taskfile.yml

@@ -0,0 +1,68 @@
+version: '3'
+
+vars:
+  PYENV: "./env"
+  PYENVSRC: "{{ .PYENV }}/src"
+  PY_PROJECT: "github.com/pulumi/pulumi/sdk/v3/go/pulumi-language-go"
+  PY_VERSION:
+    sh: cd ../../ && pulumictl get version --language python && cd sdk/go
+
+tasks:
+  ensure:
+    cmds:
+      - pipenv install --dev
+      - mkdir -p "{{ .PYENVSRC }}"
+
+  lint:
+    cmds:
+      - MYPYPATH=./stubs pipenv run mypy ./lib/pulumi --config-file=mypy.ini
+      - pipenv run pylint ./lib/pulumi --rcfile=.pylintrc
+
+  build-package:
+    cmds:
+      - rm -rf {{ .PYENVSRC }} && cp -R ./lib/. {{ .PYENVSRC }}/
+      - "sed -i.bak 's/${VERSION}/{{.PY_VERSION}}/g' {{.PYENVSRC}}/setup.py && rm {{.PYENVSRC}}/setup.py.bak"
+      - cp ../../README.md {{ .PYENVSRC }}
+      - cd {{ .PYENVSRC }} && pipenv run python setup.py build bdist_wheel --universal
+
+  build:
+    deps:
+      - build-package
+    cmds:
+      - go build -o ../../bin/pulumi-language-python -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.PY_VERSION}}" {{.PY_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-python/*.go
+    generates:
+      - ../../bin/pulumi-language-python
+    method: checksum
+
+  install-plugin:
+    cmds:
+      - cp ./cmd/pulumi-language-python-exec "{{.PULUMI_BIN}}"
+      - cp ./dist/pulumi-resource-pulumi-python "{{.PULUMI_BIN}}"
+      - cp ./dist/pulumi-analyzer-policy-python "{{.PULUMI_BIN}}"
+
+  install:
+    cmds:
+      - task: build
+      - task: install-plugin
+      - GOBIN={{.PULUMI_BIN}} go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version={{.PY_VERSION}}" {{.PY_PROJECT}}
+    sources:
+      - ./cmd/pulumi-language-python/*.go
+    generates:
+      - "{{.PULUMI_BIN}}/pulumi-language-python"
+    method: checksum
+
+  pipenv-test:
+    cmds:
+      - pipenv run pip install ./env/src
+      - pipenv run pytest lib/test --ignore lib/test/langhost/resource_thens/test_resource_thens.py
+      - pipenv run python -m unittest lib/test/langhost/resource_thens/test_resource_thens.py
+      - pushd lib/test_with_mocks ; pipenv run python -m pytest ; popd
+
+  test:
+    cmds:
+      - task: install
+      - task: pipenv-test
+      - cmd: go test -count=1 -cover -timeout 1h -tags=all -parallel {{ .TESTPARALLELISM }} ./...
+        ignore_error: true

tests/Taskfile.yml

@@ -0,0 +1,58 @@
+version: '3'
+
+tasks:
+  ensure:
+    cmds:
+      - go mod download
+    status:
+      - go mod verify
+
+  lint:
+    cmds:
+      - golangci-lint run -c ../.golangci.yml --timeout 5m
+
+  build:
+    cmds:
+      - ""
+
+  install:
+    cmds:
+      - ""
+
+  dotnet-integration:
+    cmds:
+      - PATH="{{ .PULUMI_BIN }}:$PATH" go test -count=1 -cover -timeout 1h -tags=dotnet -parallel {{ .TESTPARALLELISM }} ./integration/...
+
+  setup-go-integration-tests:
+    desc: These are the setup steps required for the Go integration tests to run
+    cmds:
+      - cd integration/construct_component/testcomponent-go && go build -o pulumi-resource-testcomponent
+      - cd integration/construct_component_plain/testcomponent-go && go build -o pulumi-resource-testcomponent
+
+  go-integration:
+    deps:
+      - setup-go-integration-tests
+    cmds:
+      - PATH="{{ .PULUMI_BIN }}:$PATH" go test -count=1 -cover -timeout 1h -tags=go -parallel {{ .TESTPARALLELISM }} ./integration/...
+
+  python-integration:
+    cmds:
+      - PATH="{{ .PULUMI_BIN }}:$PATH" go test -count=1 -cover -timeout 1h -tags=python -parallel {{ .TESTPARALLELISM }} ./integration/...
+
+  setup-nodejs-integration-tests:
+    desc: These are the setup steps required for the NodeJS integration tests to run
+    cmds:
+      - cd integration/construct_component/testcomponent && yarn install && yarn link @pulumi/pulumi && yarn run tsc
+      - cd integration/construct_component_slow/testcomponent && yarn install && yarn link @pulumi/pulumi && yarn run tsc
+      - cd integration/construct_component_plain/testcomponent && yarn install && yarn link @pulumi/pulumi && yarn run tsc
+
+  nodejs-integration:
+    deps:
+      - setup-nodejs-integration-tests
+    cmds:
+      - PATH="{{ .PULUMI_BIN }}:$PATH" go test -count=1 -cover -timeout 1h -tags=nodejs -parallel {{ .TESTPARALLELISM }} ./integration/...
+
+  integration:
+    desc: These are the non tagged integration tests in the tests repo - e.g. history / login / stack etc.
+    cmds:
+      - PATH="{{ .PULUMI_BIN }}:$PATH" go test -count=1 -cover -timeout 1h -parallel {{ .TESTPARALLELISM }} ./...