Fanout build experiment (#7628)

* Experiment with gotestsum and test timings

* Fix to locating the helper script

* Fix the code for installing gotestsum

* Try alternative installation method

* Use go to compute test stats; Python fails parsing time values

* Try version without v

* Try with fixed gorelaser config

* Fix test time correlation

* Try a stable test stat sort finally

* Use more accurate test duration aggregation

* Include python and auto-api tests in the Go timing counts

* Bring back TESTPARALLELISM

* Fix test compilation

* Only top 100 slow tests

* Try to fracture build matrix to fan out tests

* Do not run Publish Test Results on unsuppored Mac

* Auto-create test-results-dir

* Fix new flaky test by polling for logs

* Try to move native tests to their own config

* Actually skip

* Do not fail on empty test-results folder

* Try again

* Try once more

* Integration test config is the crit path - make it smaller

* Squash underutilized test configurations

* Remove the test result summary box from PR - counts now incorrec

* Remove debugging step

.github/workflows/run-build-and-acceptance-tests.yml

@@ -111,9 +111,13 @@ jobs:
         python-version: [ 3.9.x ]
         dotnet-version: [ 3.1.x ]
         node-version: [ 14.x ]
+        test-subset: [ integration, auto-and-lifecycletest, native, etc ]
     if: github.event_name == 'repository_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ${{ matrix.platform }}
     steps:
+      - name: Set PULUMI_TEST_SUBSET env var
+        run: |
+          echo "PULUMI_TEST_SUBSET=${{ matrix.test-subset }}" >> $GITHUB_ENV
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v1
         with:
@@ -154,6 +158,15 @@ jobs:
         uses: jaxxstorm/action-install-gh-release@v1.2.0
         with:
           repo: pulumi/pulumictl
+      - name: Install gotestsum
+        uses: jaxxstorm/action-install-gh-release@v1.2.0
+        with:
+          repo: gotestyourself/gotestsum
+      - name: Install goteststats
+        uses: jaxxstorm/action-install-gh-release@v1.2.0
+        with:
+          repo: t0yv0/goteststats
+          tag: v0.0.7
       - name: Ensure
         run: |
           make ensure
@@ -177,6 +190,16 @@ jobs:
           PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
           PULUMI_LOCAL_NUGET: ${{ runner.temp }}/opt/pulumi/nuget
           PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
+      - name: Summarize Test Time by Package
+        run: |
+          mkdir -p test-results
+          touch test-results/empty.json # otherwise goteststats fails below when no files match
+          goteststats -statistic pkg-time test-results/*.json
+      - name: Summarize Test Times by Indivudal Test
+        run: |
+          goteststats -statistic test-time test-results/*.json | head -n 100
+
+
   windows-build:
     name: Windows Build + Test
     strategy:

Makefile

@@ -1,7 +1,8 @@
 PROJECT_NAME := Pulumi SDK
+PROJECT_ROOT := $(realpath .)
 SUB_PROJECTS := sdk/dotnet sdk/nodejs sdk/python sdk/go
-include build/common.mk
 
+include build/common.mk
 
 PROJECT         := github.com/pulumi/pulumi/pkg/v3/cmd/pulumi
 PROJECT_PKGS    := $(shell cd ./pkg && go list ./... | grep -v /vendor/)

build/common.mk

@@ -110,8 +110,9 @@ PULUMI_BIN          := $(PULUMI_ROOT)/bin
 PULUMI_NODE_MODULES := $(PULUMI_ROOT)/node_modules
 PULUMI_NUGET        := $(PULUMI_ROOT)/nuget
 
-GO_TEST_FAST = PATH="$(PULUMI_BIN):$(PATH)" go test -short -count=1 -cover -tags=all -timeout 1h -parallel ${TESTPARALLELISM}
-GO_TEST = PATH="$(PULUMI_BIN):$(PATH)" go test -count=1 -cover -timeout 1h -tags=all -parallel ${TESTPARALLELISM}
+RUN_TESTSUITE = python3 ${PROJECT_ROOT}/scripts/run-testsuite.py
+GO_TEST_FAST = PATH="$(PULUMI_BIN):$(PATH)" python3 ${PROJECT_ROOT}/scripts/go-test.py -short -count=1 -cover -tags=all -timeout 1h -parallel ${TESTPARALLELISM}
+GO_TEST = PATH="$(PULUMI_BIN):$(PATH)" python3 $(PROJECT_ROOT)/scripts/go-test.py -count=1 -cover -timeout 1h -tags=all -parallel ${TESTPARALLELISM}
 GOPROXY = 'https://proxy.golang.org'
 
 .PHONY: default all ensure only_build only_test build lint install test_all core

scripts/go-test.py

@@ -0,0 +1,62 @@
+"""
+Wraps `go test`.
+"""
+
+from test_subsets import TEST_SUBSETS
+import os
+import pathlib
+import shutil
+import subprocess as sp
+import sys
+import uuid
+
+
+def options(options_and_packages):
+    return [o for o in options_and_packages if '/' not in o]
+
+
+def packages(options_and_packages):
+    return [o for o in options_and_packages if '/' in o]
+
+
+def filter_packages(packages, test_subset=None):
+    if test_subset is None:
+        return packages
+
+    if test_subset == 'etc':
+        s = set([])
+        for k in TEST_SUBSETS:
+            s = s | set(TEST_SUBSETS[k])
+        return [p for p in packages if p not in s]
+
+    s = set(TEST_SUBSETS[test_subset])
+    return [p for p in packages if p in s]
+
+
+root = pathlib.Path(__file__).absolute().parent.parent
+test_subset = os.environ.get('PULUMI_TEST_SUBSET', None)
+options_and_packages = sys.argv[1:]
+packages = filter_packages(packages(options_and_packages), test_subset=test_subset)
+
+
+if not packages:
+    print(f'No packages matching PULUMI_TEST_SUBSET={test_subset}')
+    sys.exit(0)
+
+
+options_and_packages = options(options_and_packages) + packages
+
+
+if shutil.which('gotestsum') is not None:
+    test_run = str(uuid.uuid4())
+
+    test_results_dir = root.joinpath('test-results')
+    if not test_results_dir.is_dir():
+        os.mkdir(str(test_results_dir))
+
+    json_file = str(test_results_dir.joinpath(f'{test_run}.json'))
+    junit_file = str(test_results_dir.joinpath(f'{test_run}.xml'))
+    sp.check_call(['gotestsum', '--jsonfile', json_file, '--junitfile', junit_file, '--'] + \
+                  options_and_packages, shell=False)
+else:
+    sp.check_call(['go', 'test'] + options_and_packages, shell=False)

scripts/run-testsuite.py

@@ -0,0 +1,45 @@
+"""
+Wraps test suite invocation shell commands to measure time and
+provide awareness of test configurations set by PULUMI_TEST_SUBSET.
+"""
+
+from test_subsets import TEST_SUBSETS
+import os
+import subprocess as sp
+import sys
+import timeit
+
+
+testsuite_name = sys.argv[1]
+testsuite_command = sys.argv[2:]
+test_subset = os.environ.get('PULUMI_TEST_SUBSET', None)
+
+
+def should_run():
+    if test_subset is None:
+        return True
+
+    if test_subset == 'etc':
+        s = set([])
+
+        for k in TEST_SUBSETS:
+            s = s | set(TEST_SUBSETS[k])
+
+        return testsuite_name not in s
+
+    s = set(TEST_SUBSETS[test_subset])
+    return testsuite_name in s
+
+
+if not should_run():
+    print(f'TESTSUITE {testsuite_name} skipped according to PULUMI_TEST_SUBSET={test_subset}')
+    sys.exit(0)
+
+
+t0 = timeit.timeit()
+try:
+    sp.check_call(testsuite_command, shell=False)
+finally:
+    t1 = timeit.timeit()
+    elapsed = t1 - t0
+    print(f'TESTSUITE {testsuite_name} completed in {elapsed}')

scripts/test_subsets.py

@@ -0,0 +1,30 @@
+"""Defines test subsets.
+
+When removing or introducing new test subsets, make sure the
+`test-subset` build matrix in `run-build-and-acceptance-tests.yml`
+matches. An implied subset `etc` will catch tests not matched by any
+explicit subset listed here.
+
+A note on the format of TEST_SUBSETS. The keys are test configuration
+names, and the values are lists of either Go packages containing the
+tests, or test suites names as passed to `run-testsuite.py`.
+
+"""
+
+TEST_SUBSETS = {
+    'integration': [
+        'github.com/pulumi/pulumi/tests/integration'
+    ],
+    'auto-and-lifecycletest': [
+        'github.com/pulumi/pulumi/sdk/v3/go/auto',
+        'github.com/pulumi/pulumi/pkg/v3/engine/lifeycletest'
+    ],
+    'native': [
+        'dotnet-test',
+        'istanbul',
+        'istanbul-with-mocks',
+        'python/lib/test',
+        'python/lib/test/langhost/resource_thens',
+        'python/lib/test_with_mocks'
+    ]
+}

sdk/dotnet/Makefile

@@ -2,6 +2,7 @@ PROJECT_NAME    := Pulumi .NET Core SDK
 LANGHOST_PKG    := github.com/pulumi/pulumi/sdk/v3/dotnet/cmd/pulumi-language-dotnet
 
 PROJECT_PKGS    := $(shell go list ./cmd...)
+PROJECT_ROOT    := $(realpath ../..)
 
 DOTNET_VERSION  := $(shell cd ../../ && pulumictl get version --language dotnet)
 
@@ -36,7 +37,7 @@ install:: build install_plugin
 
 dotnet_test:: install
 	# include the version prefix/suffix to avoid generating a separate nupkg file
-	dotnet test /p:Version=${DOTNET_VERSION}
+	$(RUN_TESTSUITE) dotnet-test dotnet test /p:Version=${DOTNET_VERSION}
 
 test_fast:: dotnet_test
 	$(GO_TEST_FAST) ${PROJECT_PKGS}
@@ -55,4 +56,3 @@ publish:: build install
 	echo "Publishing .nupkgs to nuget.org:"
 	find /opt/pulumi/nuget -name 'Pulumi*.nupkg' \
 		-exec dotnet nuget push -k ${NUGET_PUBLISH_KEY} -s https://api.nuget.org/v3/index.json {} ';'
-

sdk/go/Makefile

@@ -2,8 +2,8 @@ PROJECT_NAME     := Pulumi Go SDK
 LANGHOST_PKG     := github.com/pulumi/pulumi/sdk/v3/go/pulumi-language-go
 VERSION          := $(shell cd ../../ && pulumictl get version)
 PROJECT_PKGS     := $(shell go list ./pulumi/... ./pulumi-language-go/... ./common/... ./auto/...| grep -v /vendor/ | grep -v templates)
-
-TESTPARALLELISM := 10
+TESTPARALLELISM  := 10
+PROJECT_ROOT     := $(realpath ../..)
 
 include ../../build/common.mk
 
@@ -21,7 +21,7 @@ install:: install_plugin
 test_all:: test_fast
 
 test_fast:: install
-	go test -count=1 -cover -parallel ${TESTPARALLELISM} ${PROJECT_PKGS}
+	$(GO_TEST_FAST) ${PROJECT_PKGS}
 
 dist::
 	go install -ldflags "-X github.com/pulumi/pulumi/sdk/v3/go/common/version.Version=${VERSION}" ${LANGHOST_PKG}

sdk/go/common/workspace/plugins_install_test.go

@@ -107,7 +107,8 @@ func assertPluginInstalled(t *testing.T, dir string, plugin PluginInfo) {
 	assert.NoError(t, err)
 	assert.True(t, has)
 
-	plugins, err := getPlugins(dir)
+	skipMetadata := true
+	plugins, err := getPlugins(dir, skipMetadata)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(plugins))
 	assert.Equal(t, plugin.Name, plugins[0].Name)
@@ -255,6 +256,7 @@ func TestGetPluginsSkipsPartial(t *testing.T) {
 	assert.Error(t, err)
 	assert.False(t, has)
 
-	plugins, err := getPlugins(dir)
+	skipMetadata := true
+	plugins, err := getPlugins(dir, skipMetadata)
 	assert.Equal(t, 0, len(plugins))
 }

sdk/nodejs/Makefile

@@ -3,6 +3,7 @@ NODE_MODULE_NAME := @pulumi/pulumi
 VERSION 		 := $(shell cd ../../ && pulumictl get version --language javascript)
 
 LANGUAGE_HOST    := github.com/pulumi/pulumi/sdk/v3/nodejs/cmd/pulumi-language-nodejs
+PROJECT_ROOT     := $(realpath ../..)
 
 PROJECT_PKGS    := $(shell go list ./cmd...)
 TESTPARALLELISM := 10
@@ -42,10 +43,10 @@ install_plugin:: build
 install:: install_package install_plugin
 
 istanbul_tests:: build
-	./node_modules/.bin/istanbul test --print none _mocha -- --timeout 120000 'bin/tests/**/*.spec.js'
+	$(RUN_TESTSUITE) istanbul ./node_modules/.bin/istanbul test --print none _mocha -- --timeout 120000 'bin/tests/**/*.spec.js'
 	./node_modules/.bin/istanbul report text-summary
 	./node_modules/.bin/istanbul report text
-	./node_modules/.bin/istanbul test --print none _mocha -- 'bin/tests_with_mocks/**/*.spec.js'
+	$(RUN_TESTSUITE) istanbul-with-mocks ./node_modules/.bin/istanbul test --print none _mocha -- 'bin/tests_with_mocks/**/*.spec.js'
 
 sxs_tests:: build
 	pushd tests/sxs_ts_3.6 && yarn ; tsc ; popd

sdk/python/Makefile

@@ -2,6 +2,7 @@ PROJECT_NAME     := Pulumi Python SDK
 LANGHOST_PKG     := github.com/pulumi/pulumi/sdk/v3/python/cmd/pulumi-language-python
 VERSION          := $(shell cd ../../ && pulumictl get version)
 PYPI_VERSION 	 := $(shell cd ../../ && pulumictl get version --language python)
+PROJECT_ROOT     := $(realpath ../..)
 
 PYENV := ./env
 PYENVSRC := $(PYENV)/src
@@ -42,13 +43,13 @@ install_plugin:: build_plugin
 install:: install_package install_plugin
 
 test_fast:: build
-	go test -count=1 -cover -parallel ${TESTPARALLELISM} ${PROJECT_PKGS}
+	$(GO_TEST) ${PROJECT_PKGS}
 	pipenv run pip install ./env/src
 	# TODO the ignored test seems to fail in pytest but not unittest. Need to trackdown why
-	pipenv run pytest lib/test --ignore lib/test/langhost/resource_thens/test_resource_thens.py
-	pipenv run python -m unittest lib/test/langhost/resource_thens/test_resource_thens.py
+	$(RUN_TESTSUITE) python/lib/test pipenv run pytest lib/test --ignore lib/test/langhost/resource_thens/test_resource_thens.py
+	$(RUN_TESTSUITE) python/lib/test/langhost/resource_thens pipenv run python -m unittest lib/test/langhost/resource_thens/test_resource_thens.py
 	# Using python -m also adds lib/test_with_mocks to sys.path which avoids package resolution issues.
-	pushd lib/test_with_mocks ; pipenv run python -m pytest ; popd
+	pushd lib/test_with_mocks; $(RUN_TESTSUITE) python/lib/test_with_mocks pipenv run python -m pytest; popd
 
 test_all:: test_fast