[testing] Support gathering coverage data.

- Add infrastructure for running integration tests using a
  coverage-instrumented CLI.
- Add infrastructure for running Go tests with code coverage enabled.

All coverage profiles are emitted into a single directory. Coverage is
enabled if the `PULUMI_TEST_COVERAGE_PATH` environment variable set.

.gitignore

@@ -10,6 +10,7 @@
 coverage.cov
 *.coverprofile
 **/obj/
+coverage/
 
 **/.idea/
 *.iml

pkg/testing/integration/program.go

@@ -228,6 +228,12 @@ type ProgramTestOptions struct {
 	// file.
 	Tracing string
 
+	// If non-empty, specifies the value of the `--test.coverprofile` flag to pass to the Pulumi CLI. As with the
+	// Tracing field, the `{command}` template will expand to the current command name.
+	//
+	// If PULUMI_TEST_COVERAGE_PATH is set, this defaults to $PULUMI_TEST_COVERAGE_PATH/{command}-[random suffix].out
+	CoverProfile string
+
 	// NoParallel will opt the test out of being ran in parallel.
 	NoParallel bool
 
@@ -447,6 +453,9 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt
 	if overrides.Tracing != "" {
 		opts.Tracing = overrides.Tracing
 	}
+	if overrides.CoverProfile != "" {
+		opts.CoverProfile = overrides.CoverProfile
+	}
 	if overrides.NoParallel {
 		opts.NoParallel = overrides.NoParallel
 	}
@@ -606,6 +615,16 @@ func prepareProgram(t *testing.T, opts *ProgramTestOptions) {
 	if opts.Tracing == "" {
 		opts.Tracing = os.Getenv("PULUMI_TEST_TRACE_ENDPOINT")
 	}
+
+	if opts.CoverProfile == "" {
+		if cov := os.Getenv("PULUMI_TEST_COVERAGE_PATH"); cov != "" {
+			var b [4]byte
+			if _, err := cryptorand.Read(b[:]); err != nil {
+				t.Errorf("could not read random bytes: %v", err)
+			}
+			opts.CoverProfile = filepath.Join(cov, "{command}-"+hex.EncodeToString(b[:])+".cov")
+		}
+	}
 }
 
 // ProgramTest runs a lifecycle of Pulumi commands in a program working directory, using the `pulumi` and `yarn`
@@ -746,6 +765,9 @@ func (pt *ProgramTester) pulumiCmd(name string, args []string) ([]string, error)
 	if tracing := pt.opts.Tracing; tracing != "" {
 		cmd = append(cmd, "--tracing", strings.ReplaceAll(tracing, "{command}", name))
 	}
+	if cov := pt.opts.CoverProfile; cov != "" {
+		cmd = append(cmd, "--test.coverprofile", strings.ReplaceAll(cov, "{command}", name))
+	}
 	return cmd, nil
 }
 

scripts/go-test.py

@@ -44,7 +44,13 @@ if not packages:
     sys.exit(0)
 
 
-options_and_packages = options(options_and_packages) + packages
+options = options(options_and_packages)
+cov = os.environ.get('PULUMI_TEST_COVERAGE_PATH', None)
+if cov is not None:
+    options = options + [f'-coverprofile={cov}/go-test-{os.urandom(4).hex()}.cov', '-coverpkg=github.com/pulumi/pulumi/pkg/v3/...,github.com/pulumi/pulumi/sdk/v3/...']
+
+
+options_and_packages = options + packages
 
 
 if shutil.which('gotestsum') is not None: