Prototype schema coverage tracker

.gitignore

@@ -47,3 +47,6 @@ pulumi-python3-shim.cmd
 pulumi-python-shim.cmd
 pulumi-analyzer-policy.cmd
 pulumi-analyzer-policy-python.cmd
+
+
+__pycache__

pkg/codegen/coverage/coverage_test.py

@@ -0,0 +1,263 @@
+import json
+import subprocess as sp
+import jsonschema
+import jsonschema.exceptions
+
+
+def load_json_file(path):
+    with open(path) as fp:
+        return json.load(fp)
+
+
+def parse_ref(j):
+    if type(j) == dict and '$ref' in j:
+        return j['$ref']
+
+    return None
+
+
+def resolve_ref(root, ref):
+    parts = ref.split('/')
+    assert parts[0] == '#'
+    for p in parts[1:]:
+        root = root[p]
+    return root
+
+
+ignored_keys = [
+    '$defs',
+    '$id',
+    '$schema',
+    'const',
+    'description',
+    'enum', # TODO enum coverage may be interesting
+    'format',
+    'pattern',
+    'propertyNames',
+    'required',
+    'minItems',
+    'title',
+]
+
+
+understood_keys = [
+    'type',
+    'items',
+    'properties',
+    'additionalProperties',
+    'allOf',
+    'oneOf',
+]
+
+
+def is_valid(root_schema, schema, json):
+    s = {}
+
+    for k in schema:
+        s[k] = schema[k]
+
+    s['$defs'] = root_schema['$defs']
+
+    try:
+        jsonschema.validate(schema=s, instance=json)
+        return True
+    except jsonschema.exceptions.ValidationError as e:
+        return False
+
+
+PATHS = set([])
+"""Collects covered schema paths."""
+
+
+POSSIBLE_PATHS = set([])
+"""Collects all possible  schema paths."""
+
+
+def find_all_possible_paths(root_schema, schema, path=None):
+    if path is None:
+        path = '#'
+
+    if path in POSSIBLE_PATHS:
+        return  # seen already
+
+    POSSIBLE_PATHS.add(path)
+
+    ref = parse_ref(schema)
+    if ref is not None:
+        # print(f'Forwarding {path} to {ref}')
+        find_all_possible_paths(root_schema, resolve_ref(root_schema, ref), path=ref)
+        return
+
+    if 'const' in schema:
+        return
+
+    if 'type' not in schema:
+        raise Exception(f"Cannot parse type spec: 'type' field missing: {schema}")
+
+    def prim(t):
+        return t in ['string', 'boolean', 'number', 'integer']
+
+    if prim(schema['type']):
+        return
+
+    if type(schema['type']) == list and all(prim(t) for t in schema['type']):
+        return
+
+    if schema['type'] == 'array':
+        find_all_possible_paths(root_schema, schema['items'], f'{path}/items')
+        return
+
+    if schema['type'] == 'object':
+
+        properties = schema.get('properties', {})
+        for prop_name, prop_schema in properties.items():
+            find_all_possible_paths(root_schema, prop_schema, f'{path}/{prop_name}')
+
+        aProperties = schema.get('additionalProperties', False)
+        if type(aProperties) != bool:
+            find_all_possible_paths(root_schema,
+                                    aProperties,
+                                    f'{path}/additionalProperties')
+
+        all_of = schema.get('allOf', [])
+        if all_of:
+            for i, ty in enumerate(all_of):
+                find_all_possible_paths(root_schema, ty, path=f'{path}/allOf[{i}]')
+
+        one_of = schema.get('oneOf', [])
+        if one_of:
+            for i, ty in enumerate(one_of):
+                find_all_possible_paths(root_schema, ty, path=f'{path}/oneOf[{i}]')
+
+        return
+
+    raise Exception(f'Cannot understand schema: {schema}')
+
+
+def validate(root_schema, schema, json, path=None):
+    if path is None:
+        path = '#'
+
+    PATHS.add(path)
+
+    # print(f'validating path={path}')
+
+    ref = parse_ref(schema)
+    if ref is not None:
+        # print(f'Forwarding {path} to {ref}')
+        validate(root_schema, resolve_ref(root_schema, ref), json, path=ref)
+        return
+
+    for key in schema:
+        if key not in ignored_keys + understood_keys:
+            print(f'Confusing KEY={key} AT path={path}')
+
+    if 'const' in schema:
+        return
+
+    if 'type' not in schema:
+        raise Exception(f"Cannot parse type spec: 'type' field missing: {schema}")
+
+    def prim(t):
+        return t in  ['string', 'boolean', 'number', 'integer']
+
+    if prim(schema['type']):
+        return
+
+    if type(schema['type']) == list and all(prim(t) for t in schema['type']):
+        return
+
+    if schema['type'] == 'array':
+        t = schema['items']
+
+        if type(json) == list:
+            for v in json:
+                validate(root_schema, t, v, path=f'{path}/items')
+
+        return
+
+    if schema['type'] == 'object':
+
+        properties = schema.get('properties', {})
+        for prop_name, prop_schema in properties.items():
+            if prop_name in json:
+                validate(root_schema, prop_schema, json[prop_name], f'{path}/{prop_name}')
+
+        aProperties = schema.get('additionalProperties', False)
+        if aProperties != False:
+            if type(json) == dict:
+                for value_key, value in json.items():
+                    if value_key not in properties:
+                        validate(root_schema,
+                                 schema['additionalProperties'],
+                                 value,
+                                 path=f'{path}/additionalProperties')
+
+        allOf = schema.get('allOf', [])
+        for i, ty in enumerate(allOf):
+            validate(root_schema, ty, json, path=f'{path}/allOf[{i}]')
+
+        one_of = schema.get('oneOf', [])
+        if one_of:
+            matches = [
+                (f'{path}/oneOf[{i}]', ty)
+                for (i, ty) in enumerate(one_of)
+                if is_valid(root_schema, ty, json)
+            ]
+
+            if len(matches) != 1:
+                print(f'WARN: when validating against {path}/oneOf expected exactly 1 match, got {len(matches)}')
+                # print(json)
+            else:
+                (path, ty) = matches[0]
+                validate(root_schema, ty, json, path=path)
+
+        return
+
+    raise Exception(f'Cannot understand schema: {schema}')
+
+
+
+meta_schema = load_json_file('../schema/pulumi.json')
+find_all_possible_paths(meta_schema, meta_schema)
+
+
+schema_corpus = [{'file': line, 'schema': load_json_file(line)}
+                 for line in sp.check_output(
+                         'ls ../internal/test/testdata/*/schema.json',
+                         shell=True).decode('utf-8').split('\n')
+                 if line]
+
+
+for exemplar in schema_corpus:
+    print('Parsing', exemplar['file'])
+    valid = is_valid(meta_schema, meta_schema,  exemplar['schema'])
+    print('Valid overall: ', valid)
+    validate(meta_schema, meta_schema, exemplar['schema'])
+
+
+print('=' * 80)
+print('Covered meta-schema paths')
+print('=' * 80)
+for p in sorted(p for p in PATHS):
+    print(p)
+
+print()
+print()
+
+print('=' * 80)
+print('Uncovered meta-schema paths')
+print('=' * 80)
+for p in sorted(p for p in POSSIBLE_PATHS - PATHS):
+    print(p)
+
+print()
+print()
+
+print('=' * 80)
+print('Statistics')
+print('=' * 80)
+
+print('possible        ', len(POSSIBLE_PATHS))
+print('covered         ', len(PATHS))
+print('covered/possible', len(PATHS)/len(POSSIBLE_PATHS))

pkg/codegen/coverage/requirements.txt

@@ -0,0 +1 @@
+jsonschema

pkg/codegen/docs/gen_test.go

@@ -156,12 +156,12 @@ func initTestPackageSpec(t *testing.T) {
 {{% /example %}}
 {{% /examples %}}
 
-## Import 
+## Import
 
 The import docs would be here
 
 ` + codeFence + `sh
-$ pulumi import prov:module/resource:Resource test test		
+$ pulumi import prov:module/resource:Resource test test
 ` + codeFence + `
 `,
 				},
@@ -412,5 +412,8 @@ func generatePackage(tool string, pkg *schema.Package, extraFiles map[string][]b
 
 func TestGeneratePackage(t *testing.T) {
 	// TODO: do we have a compile step on templates?
-	test.TestSDKCodegen(t, "docs", generatePackage, func(*testing.T, string) {})
+	test.TestSDKCodegen(t, &test.SDKCodegenOptions{
+		Language:   "docs",
+		GenPackage: generatePackage,
+	})
 }

pkg/codegen/dotnet/gen_test.go

@@ -1,8 +1,6 @@
 package dotnet
 
 import (
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -15,21 +13,30 @@ import (
 )
 
 func TestGeneratePackage(t *testing.T) {
-	test.TestSDKCodegen(t, "dotnet", GeneratePackage, typeCheckGeneratedPackage)
+	test.TestSDKCodegen(t, &test.SDKCodegenOptions{
+		Language:   "dotnet",
+		GenPackage: generatePackageWithVersion,
+		Checks: map[string]test.CodegenCheck{
+			"dotnet/compile": typeCheckGeneratedPackage,
+		},
+	})
+}
+
+// TODO replace this with GeneratePackage when https://github.com/pulumi/pulumi/pull/7938 lands.
+func generatePackageWithVersion(
+	tool string,
+	pkg *schema.Package,
+	extraFiles map[string][]byte) (map[string][]byte, error) {
+	if extraFiles == nil {
+		extraFiles = make(map[string][]byte)
+	}
+	extraFiles["version.txt"] = []byte("0.0.0\n")
+	return GeneratePackage(tool, pkg, extraFiles)
 }
 
 func typeCheckGeneratedPackage(t *testing.T, pwd string) {
 	var err error
 	var dotnet string
-
-	// TODO remove when https://github.com/pulumi/pulumi/pull/7938 lands
-	version := "0.0.0\n"
-	err = os.WriteFile(filepath.Join(pwd, "version.txt"), []byte(version), 0600)
-	if !os.IsExist(err) {
-		require.NoError(t, err)
-	}
-	// endTODO
-
 	dotnet, err = executable.FindExecutable("dotnet")
 	require.NoError(t, err)
 	cmdOptions := integration.ProgramTestOptions{}

pkg/codegen/go/gen_test.go

@@ -6,24 +6,20 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
-	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test"
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant"
-	tree "github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant/tree/v1"
 	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
-	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
 func TestInputUsage(t *testing.T) {
@@ -66,38 +62,76 @@ func TestGoPackageName(t *testing.T) {
 
 func TestGeneratePackage(t *testing.T) {
 	generatePackage := func(tool string, pkg *schema.Package, files map[string][]byte) (map[string][]byte, error) {
+
+		for f := range files {
+			t.Logf("Ignoring extraFile %s", f)
+		}
+
 		return GeneratePackage(tool, pkg)
 	}
-	test.TestSDKCodegen(t, "go", generatePackage, typeCheckGeneratedPackage)
+	test.TestSDKCodegen(t, &test.SDKCodegenOptions{
+		Language:   "go",
+		GenPackage: generatePackage,
+		Checks: map[string]test.CodegenCheck{
+			"go/compile": typeCheckGeneratedPackage,
+			"go/test":    testGeneratedPackage,
+		},
+	})
 }
 
-func typeCheckGeneratedPackage(t *testing.T, pwd string) {
-	var err error
-	var ex string
-	ex, err = executable.FindExecutable("go")
-	require.NoError(t, err)
-
-	// go SDKs live in their own folder:
-	// typecheck/go/$NAME/$FILES
-	// where FILES contains all the project files (including go.mod)
+func inferModuleName(codeDir string) string {
+	// For example for this path:
 	//
-	// This is not true when `package.language.go.rootPackageName` is set.
-	dir, err := ioutil.ReadDir(pwd)
-	require.NoError(t, err)
-	root := pwd
-	if len(dir) == 1 {
-		require.True(t, dir[0].IsDir())
-		root = filepath.Join(pwd, dir[0].Name())
-	}
-	t.Logf("*** Testing go in dir: %q ***", root)
+	// codeDir = "../internal/test/testdata/external-resource-schema/go/"
+	//
+	// We will generate "$codeDir/go.mod" using
+	// `external-resource-schema` as the module name so that it
+	// can compile independently.
+	return filepath.Base(filepath.Dir(codeDir))
+}
 
+func typeCheckGeneratedPackage(t *testing.T, codeDir string) {
+	sdk, err := filepath.Abs(filepath.Join("..", "..", "..", "sdk"))
+	require.NoError(t, err)
+
+	goExe, err := executable.FindExecutable("go")
+	require.NoError(t, err)
+
+	// Remove existing `go.mod` first otherise `go mod init` fails.
+	goMod := filepath.Join(codeDir, "go.mod")
+	alreadyHaveGoMod, err := test.PathExists(goMod)
+
+	require.NoError(t, err)
+	if alreadyHaveGoMod {
+		err := os.Remove(goMod)
+		require.NoError(t, err)
+	}
+
+	runCommand(t, "go_mod_init", codeDir, goExe, "mod", "init", inferModuleName(codeDir))
+	replacement := fmt.Sprintf("github.com/pulumi/pulumi/sdk/v3=%s", sdk)
+	runCommand(t, "go_mod_edit", codeDir, goExe, "mod", "edit", "-replace", replacement)
+	runCommand(t, "go_mod_tidy", codeDir, goExe, "mod", "tidy")
+	runCommand(t, "go_build", codeDir, goExe, "build", "-v", "all")
+}
+
+func testGeneratedPackage(t *testing.T, codeDir string) {
+	goExe, err := executable.FindExecutable("go")
+	require.NoError(t, err)
+
+	runCommand(t, "go-test", codeDir, goExe, "test", fmt.Sprintf("%s/...", inferModuleName(codeDir)))
+}
+
+func runCommand(t *testing.T, name string, cwd string, executable string, args ...string) {
+	wd, err := filepath.Abs(cwd)
+	require.NoError(t, err)
 	var stdout, stderr bytes.Buffer
-	cmdOptions := integration.ProgramTestOptions{Stderr: &stderr, Stdout: &stdout}
-	// We don't want to corrupt the global go.mod and go.sum with packages we
-	// don't actually depend on. For this, we need to have each go package be
-	// it's own module.
-	err = integration.RunCommand(t, "mod init",
-		[]string{ex, "mod", "init", "github.com/pulumi/test/internal"}, root, &cmdOptions)
+	cmdOptions := integration.ProgramTestOptions{Stderr: &stderr, Stdout: &stdout, Verbose: true}
+	err = integration.RunCommand(t,
+		name,
+		append([]string{executable}, args...),
+		wd,
+		&cmdOptions)
+	require.NoError(t, err)
 	if err != nil {
 		stdout := stdout.String()
 		stderr := stderr.String()
@@ -109,10 +143,6 @@ func typeCheckGeneratedPackage(t *testing.T, pwd string) {
 		}
 		t.FailNow()
 	}
-	err = integration.RunCommand(t, "get", []string{ex, "get"}, root, &cmdOptions)
-	require.NoError(t, err)
-	err = integration.RunCommand(t, "go build", []string{ex, "build"}, root, &cmdOptions)
-	require.NoError(t, err)
 }
 
 func TestGenerateTypeNames(t *testing.T) {
@@ -135,177 +165,6 @@ func TestGenerateTypeNames(t *testing.T) {
 	})
 }
 
-type mocks int
-
-func (mocks) NewResource(args pulumi.MockResourceArgs) (string, resource.PropertyMap, error) {
-	return args.Name + "_id", args.Inputs, nil
-}
-
-func (mocks) Call(args pulumi.MockCallArgs) (resource.PropertyMap, error) {
-	return args.Args, nil
-}
-
-func TestEnumUsage(t *testing.T) {
-	t.Run("Success", func(t *testing.T) {
-		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
-			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
-				Container: &plant.ContainerArgs{
-					Color:    plant.ContainerColorRed,
-					Material: pulumi.String("ceramic"),
-					Size:     plant.ContainerSizeFourInch,
-				},
-				Farm: tree.Farm_Plants_R_Us,
-				Type: tree.RubberTreeVarietyRuby,
-			})
-			require.NoError(t, err)
-			require.NotNil(t, rubberTree)
-			var wg sync.WaitGroup
-			wg.Add(1)
-			pulumi.All(
-				rubberTree.URN(),
-				rubberTree.Container.Material(),
-				rubberTree.Container.Color(),
-				rubberTree.Container.Size(),
-				rubberTree.Container.Brightness(),
-				rubberTree.Type,
-			).ApplyT(func(all []interface{}) error {
-				urn := all[0].(pulumi.URN)
-				material := all[1].(*string)
-				color := all[2].(*string)
-				size := all[3].(*plant.ContainerSize)
-				brightness := all[4].(*plant.ContainerBrightness)
-				typ := all[5].(tree.RubberTreeVariety)
-				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
-				assert.Equal(t, *color, "red", "unexpected color on resource: %v", urn)
-				assert.Equal(t, *size, plant.ContainerSizeFourInch, "unexpected size on resource: %v", urn)
-				assert.Nil(t, brightness)
-				assert.Equal(t, typ, tree.RubberTreeVarietyRuby, "unexpected type on resource: %v", urn)
-				wg.Done()
-				return nil
-			})
-			wg.Wait()
-			return nil
-		}, pulumi.WithMocks("project", "stack", mocks(0))))
-	})
-
-	t.Run("StringsForRelaxedEnum", func(t *testing.T) {
-		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
-			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
-				Container: plant.ContainerArgs{
-					Color:    pulumi.String("Magenta"),
-					Material: pulumi.String("ceramic"),
-					Size:     plant.ContainerSize(22),
-				},
-				Farm: tree.Farm_Plants_R_Us,
-				Type: tree.RubberTreeVarietyRuby,
-			})
-			require.NoError(t, err)
-			require.NotNil(t, rubberTree)
-			var wg sync.WaitGroup
-			wg.Add(1)
-			pulumi.All(
-				rubberTree.URN(),
-				rubberTree.Container.Material(),
-				rubberTree.Container.Color(),
-				rubberTree.Container.Size(),
-				rubberTree.Type,
-			).ApplyT(func(all []interface{}) error {
-				urn := all[0].(pulumi.URN)
-				material := all[1].(*string)
-				color := all[2].(*string)
-				size := all[3].(*plant.ContainerSize)
-				typ := all[4].(tree.RubberTreeVariety)
-				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
-				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
-				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
-				assert.Equal(t, typ, tree.RubberTreeVarietyRuby, "unexpected type on resource: %v", urn)
-				wg.Done()
-				return nil
-			})
-			wg.Wait()
-			return nil
-		}, pulumi.WithMocks("project", "stack", mocks(1))))
-	})
-
-	t.Run("StringsForStrictEnum", func(t *testing.T) {
-		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
-			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
-				Container: plant.ContainerArgs{
-					Color:    pulumi.String("Magenta"),
-					Material: pulumi.String("ceramic"),
-					Size:     plant.ContainerSize(22),
-				},
-				Farm: tree.Farm_Plants_R_Us,
-				Type: tree.RubberTreeVarietyBurgundy,
-			})
-			require.NoError(t, err)
-			require.NotNil(t, rubberTree)
-			var wg sync.WaitGroup
-			wg.Add(1)
-			pulumi.All(
-				rubberTree.URN(),
-				rubberTree.Container.Material(),
-				rubberTree.Container.Color(),
-				rubberTree.Container.Size(),
-				rubberTree.Type,
-			).ApplyT(func(all []interface{}) error {
-				urn := all[0].(pulumi.URN)
-				material := all[1].(*string)
-				color := all[2].(*string)
-				size := all[3].(*plant.ContainerSize)
-				typ := all[4].(tree.RubberTreeVariety)
-				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
-				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
-				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
-				assert.Equal(t, typ, tree.RubberTreeVarietyBurgundy, "unexpected type on resource: %v", urn)
-				wg.Done()
-				return nil
-			})
-			wg.Wait()
-			return nil
-		}, pulumi.WithMocks("project", "stack", mocks(1))))
-	})
-
-	t.Run("EnumOutputs", func(t *testing.T) {
-		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
-			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
-				Container: plant.ContainerArgs{
-					Color:    plant.ContainerColor("Magenta").ToContainerColorOutput().ToStringOutput(),
-					Material: pulumi.String("ceramic").ToStringOutput(),
-					Size:     plant.ContainerSize(22).ToContainerSizeOutput(),
-				},
-				Farm: tree.Farm_Plants_R_Us.ToFarmPtrOutput().ToStringPtrOutput(),
-				Type: tree.RubberTreeVarietyBurgundy.ToRubberTreeVarietyOutput(),
-			})
-			require.NoError(t, err)
-			require.NotNil(t, rubberTree)
-			var wg sync.WaitGroup
-			wg.Add(1)
-			pulumi.All(
-				rubberTree.URN(),
-				rubberTree.Container.Material(),
-				rubberTree.Container.Color(),
-				rubberTree.Container.Size(),
-				rubberTree.Type,
-			).ApplyT(func(all []interface{}) error {
-				urn := all[0].(pulumi.URN)
-				material := all[1].(*string)
-				color := all[2].(*string)
-				size := all[3].(*plant.ContainerSize)
-				typ := all[4].(tree.RubberTreeVariety)
-				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
-				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
-				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
-				assert.Equal(t, typ, tree.RubberTreeVarietyBurgundy, "unexpected type on resource: %v", urn)
-				wg.Done()
-				return nil
-			})
-			wg.Wait()
-			return nil
-		}, pulumi.WithMocks("project", "stack", mocks(1))))
-	})
-}
-
 func TestGenerateOutputFuncs(t *testing.T) {
 	testDir := filepath.Join("..", "internal", "test", "testdata", "output-funcs")
 

pkg/codegen/internal/test/helpers.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -76,7 +77,11 @@ func LoadFiles(dir, lang string, files []string) (map[string][]byte, error) {
 	return result, nil
 }
 
-func loadDirectory(fs map[string][]byte, root, path string) error {
+// Recursively loads files from a directory into the `fs` map. Ignores
+// entries that match `ignore(path)==true`, also skips descending into
+// directories that are ignored. This is useful for example to avoid
+// `node_modules`.
+func loadDirectory(fs map[string][]byte, root, path string, ignore func(path string) bool) error {
 	entries, err := os.ReadDir(path)
 	if err != nil {
 		return err
@@ -84,8 +89,12 @@ func loadDirectory(fs map[string][]byte, root, path string) error {
 
 	for _, e := range entries {
 		entryPath := filepath.Join(path, e.Name())
-		if e.IsDir() {
-			if err = loadDirectory(fs, root, entryPath); err != nil {
+		relativeEntryPath := entryPath[len(root)+1:]
+		baseName := filepath.Base(relativeEntryPath)
+		if ignore != nil && (ignore(relativeEntryPath) || ignore(baseName)) {
+			// pass
+		} else if e.IsDir() {
+			if err = loadDirectory(fs, root, entryPath, ignore); err != nil {
 				return err
 			}
 		} else {
@@ -93,8 +102,7 @@ func loadDirectory(fs map[string][]byte, root, path string) error {
 			if err != nil {
 				return err
 			}
-			name := filepath.ToSlash(entryPath[len(root)+1:])
-
+			name := filepath.ToSlash(relativeEntryPath)
 			fs[name] = contents
 		}
 	}
@@ -102,12 +110,119 @@ func loadDirectory(fs map[string][]byte, root, path string) error {
 	return nil
 }
 
+// Removes files from directory recursively unless the are ignored.
+func removeFilesFromDirUnlessIgnored(root, path string, ignore func(path string) bool) error {
+	entries, err := os.ReadDir(path)
+	if err != nil {
+		return err
+	}
+
+	for _, e := range entries {
+		entryPath := filepath.Join(path, e.Name())
+		relativeEntryPath := entryPath[len(root)+1:]
+
+		if ignore != nil && ignore(relativeEntryPath) {
+			// pass
+		} else if e.IsDir() {
+			if err = removeFilesFromDirUnlessIgnored(root, entryPath, ignore); err != nil {
+				return err
+			}
+		} else {
+			if err = os.Remove(path); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+
+	if err == nil {
+		return true, nil
+	}
+
+	return false, err
+}
+
+// Reads `.sdkcodegenignore` file if present to use as loadDirectory ignore func.
+func loadIgnoreMap(dir string) (func(path string) bool, error) {
+
+	load1 := func(dir string, ignoredPathSet map[string]bool) error {
+		p := filepath.Join(dir, ".sdkcodegenignore")
+
+		gotIgnore, err := PathExists(p)
+		if err != nil {
+			return err
+		}
+
+		if gotIgnore {
+			contents, err := os.ReadFile(p)
+			if err != nil {
+				return err
+			}
+			for _, s := range strings.Split(string(contents), "\n") {
+				s = strings.Trim(s, " \r\n\t")
+
+				if s != "" {
+					ignoredPathSet[s] = true
+				}
+			}
+		}
+		return nil
+	}
+
+	loadAll := func(dir string, ignoredPathSet map[string]bool) error {
+		for {
+			atTopOfRepo, err := PathExists(filepath.Join(dir, ".git"))
+			if err != nil {
+				return err
+			}
+
+			err = load1(dir, ignoredPathSet)
+			if err != nil {
+				return err
+			}
+
+			if atTopOfRepo || dir == "." {
+				return nil
+			}
+
+			dir = filepath.Dir(dir)
+		}
+	}
+
+	ignoredPathSet := make(map[string]bool)
+	err := loadAll(dir, ignoredPathSet)
+	if err != nil {
+		return nil, err
+	}
+
+	return func(path string) bool {
+		path = strings.ReplaceAll(path, "\\", "/")
+		_, ignoredPath := ignoredPathSet[path]
+		return ignoredPath
+	}, nil
+}
+
 // LoadBaseline loads the contents of the given baseline directory.
 func LoadBaseline(dir, lang string) (map[string][]byte, error) {
 	dir = filepath.Join(dir, lang)
 
 	fs := map[string][]byte{}
-	if err := loadDirectory(fs, dir, dir); err != nil {
+
+	ignore, err := loadIgnoreMap(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := loadDirectory(fs, dir, dir, ignore); err != nil {
 		return nil, err
 	}
 	return fs, nil
@@ -117,14 +232,20 @@ func LoadBaseline(dir, lang string) (map[string][]byte, error) {
 func ValidateFileEquality(t *testing.T, actual, expected map[string][]byte) bool {
 	ok := true
 	for name, file := range expected {
-		if !assert.Contains(t, actual, name) || !assert.Equal(t, string(file), string(actual[name]), name) {
-			t.Logf("%s did not agree", name)
+		_, inActual := actual[name]
+		if inActual {
+			if !assert.Equal(t, string(file), string(actual[name]), name) {
+				t.Logf("%s did not agree", name)
+				ok = false
+			}
+		} else {
+			t.Logf("File %s was expected but is missing from the actual fileset", name)
 			ok = false
 		}
 	}
 	for name := range actual {
-		if _, has := expected[name]; !has {
-			t.Logf("missing data for %s", name)
+		if _, inExpected := expected[name]; !inExpected {
+			t.Logf("File %s from the actual fileset was not expected", name)
 			ok = false
 		}
 	}
@@ -142,28 +263,20 @@ func RewriteFilesWhenPulumiAccept(t *testing.T, dir, lang string, actual map[str
 	baseline := filepath.Join(dir, lang)
 
 	// Remove the baseline directory's current contents.
-	entries, err := os.ReadDir(baseline)
+	_, err := os.ReadDir(baseline)
 	switch {
 	case err == nil:
-		for _, e := range entries {
-			err = os.RemoveAll(filepath.Join(baseline, e.Name()))
-			require.NoError(t, err)
-		}
+		ignore, err := loadIgnoreMap(baseline)
+		require.NoError(t, err)
+		err = removeFilesFromDirUnlessIgnored(baseline, baseline, ignore)
+		require.NoError(t, err)
 	case os.IsNotExist(err):
 		// OK
 	default:
 		require.NoError(t, err)
 	}
 
-	WriteTestFiles(t, dir, lang, actual)
-
-	return true
-}
-
-// WriteTestFiles writes out the files generated by GeneratePackage to a directory.
-func WriteTestFiles(t *testing.T, dir, lang string, files map[string][]byte) {
-	var err error
-	for file, bytes := range files {
+	for file, bytes := range actual {
 		relPath := filepath.FromSlash(file)
 		path := filepath.Join(dir, lang, relPath)
 
@@ -174,6 +287,8 @@ func WriteTestFiles(t *testing.T, dir, lang string, files map[string][]byte) {
 		err = ioutil.WriteFile(path, bytes, 0600)
 		require.NoError(t, err)
 	}
+
+	return true
 }
 
 // CheckAllFilesGenerated ensures that the set of expected and actual files generated

pkg/codegen/internal/test/sdk_driver.go

@@ -1,28 +1,40 @@
 package test
 
 import (
+	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"sort"
+	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/pulumi/pulumi/pkg/v3/codegen"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 )
 
-type ThenFunc func(t *testing.T, testDir string)
+// Defines an extra check logic that accepts the directory with the
+// generated code, typically `$TestDir/$test.Directory/$language`.
+type CodegenCheck func(t *testing.T, codedir string)
 
 type sdkTest struct {
-	Directory        string
-	Description      string
+	Directory   string
+	Description string
+
+	// Extra checks for this test. They keys of this map
+	// are of the form "$language/$check" such as "go/compile".
+	Checks map[string]CodegenCheck
+
+	// Skip checks, identified by "$language/$check".
+	Skip codegen.StringSet
+
+	// Do not compile the generated code for the languages in this set.
+	// This is a helper form of `Skip`.
 	SkipCompileCheck codegen.StringSet
-	Then             map[string]ThenFunc
 }
 
 const (
+	// python = "python"
 	nodejs = "nodejs"
 	dotnet = "dotnet"
 	golang = "go"
@@ -59,17 +71,6 @@ var sdkTests = []sdkTest{
 	{
 		Directory:   "resource-args-python",
 		Description: "Resource args with same named resource and type",
-		Then: map[string]ThenFunc{
-			"go": func(t *testing.T, testDir string) {
-				cmd := exec.Command("go", "test", "./...")
-				cmd.Dir = filepath.Join(testDir, "go-program")
-
-				out, err := cmd.CombinedOutput()
-				if !assert.NoError(t, err) {
-					t.Logf("output: %v", string(out))
-				}
-			},
-		},
 	},
 	{
 		Directory:   "simple-enum-schema",
@@ -117,7 +118,18 @@ var sdkTests = []sdkTest{
 	},
 }
 
-type checkPackageSignature = func(t *testing.T, pwd string)
+type SDKCodegenOptions struct {
+	// Name of the programming language.
+	Language string
+
+	// Language-aware code generator; such as `GeneratePackage`.
+	// from `codgen/dotnet`.
+	GenPackage GenPkgSignature
+
+	// Extra checks for all the tests. They keys of this map are
+	// of the form "$language/$check" such as "go/compile".
+	Checks map[string]CodegenCheck
+}
 
 // TestSDKCodegen runs the complete set of SDK code generation tests against a particular language's code
 // generator. It also verifies that the generated code is structurally sound.
@@ -133,7 +145,7 @@ type checkPackageSignature = func(t *testing.T, pwd string)
 //
 // The schema is the only piece that must be manually authored. Once the schema has been written, the expected outputs
 // can be generated by running `PULUMI_ACCEPT=true go test ./..." from the `pkg/codegen` directory.
-func TestSDKCodegen(t *testing.T, language string, genPackage GenPkgSignature, checkPackage checkPackageSignature) {
+func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) {
 	testDir := filepath.Join("..", "internal", "test", "testdata")
 
 	for _, tt := range sdkTests {
@@ -145,22 +157,11 @@ func TestSDKCodegen(t *testing.T, language string, genPackage GenPkgSignature, c
 				schemaPath = filepath.Join(dirPath, "schema.yaml")
 			}
 
-			files, err := GeneratePackageFilesFromSchema(schemaPath, genPackage)
+			files, err := GeneratePackageFilesFromSchema(schemaPath, opts.GenPackage)
 			require.NoError(t, err)
 
-			// Check output is valid code (will type-check). If code will not
-			// type-check, we don't allow the user to run PULUMI_ACCEPT=true and
-			// replace the test files.
-			if !tt.SkipCompileCheck.Has(language) {
-				typeCheckPath := filepath.Join(dirPath, "typecheck")
-				langTypeCheckPath := filepath.Join(typeCheckPath, language)
-				contract.IgnoreError(os.RemoveAll(langTypeCheckPath))
-				WriteTestFiles(t, typeCheckPath, language, files)
-				checkPackage(t, langTypeCheckPath)
-			}
-
-			if !RewriteFilesWhenPulumiAccept(t, dirPath, language, files) {
-				expectedFiles, err := LoadBaseline(dirPath, language)
+			if !RewriteFilesWhenPulumiAccept(t, dirPath, opts.Language, files) {
+				expectedFiles, err := LoadBaseline(dirPath, opts.Language)
 				require.NoError(t, err)
 
 				if !ValidateFileEquality(t, files, expectedFiles) {
@@ -168,8 +169,60 @@ func TestSDKCodegen(t *testing.T, language string, genPackage GenPkgSignature, c
 				}
 			}
 
-			if then, ok := tt.Then[language]; ok {
-				then(t, dirPath)
+			// Merge language-specific global and
+			// test-specific checks, with test-specific
+			// having precedence.
+			allChecks := make(map[string]CodegenCheck)
+			for k, v := range opts.Checks {
+				allChecks[k] = v
+			}
+			for k, v := range tt.Checks {
+				allChecks[k] = v
+			}
+
+			// Define check filter.
+			shouldSkipCheck := func(check string) bool {
+
+				// Only language-specific checks.
+				if !strings.HasPrefix(check, opts.Language+"/") {
+					return true
+				}
+
+				// Obey SkipCompileCheck to skip compile and test targets.
+				if tt.SkipCompileCheck != nil &&
+					tt.SkipCompileCheck.Has(opts.Language) &&
+					(check == fmt.Sprintf("%s/compile", opts.Language) ||
+						check == fmt.Sprintf("%s/test", opts.Language)) {
+					return true
+				}
+
+				// Obey Skip.
+				if tt.Skip != nil && tt.Skip.Has(check) {
+					return true
+				}
+
+				return false
+			}
+
+			// Sort the checks in alphabetical order.
+			var checkOrder []string
+			for check := range allChecks {
+				checkOrder = append(checkOrder, check)
+			}
+			sort.Strings(checkOrder)
+
+			codeDir := filepath.Join(dirPath, opts.Language)
+
+			// Perform the checks.
+			for _, checkVar := range checkOrder {
+				check := checkVar
+				t.Run(check, func(t *testing.T) {
+					if shouldSkipCheck(check) {
+						t.Skip()
+					}
+					checkFun := allChecks[check]
+					checkFun(t, codeDir)
+				})
 			}
 		})
 	}

pkg/codegen/internal/test/testdata/.gitignore

@@ -0,0 +1,3 @@
+package-lock.json
+go.sum
+go.mod

pkg/codegen/internal/test/testdata/.sdkcodegenignore

@@ -0,0 +1,10 @@
+__pycache__
+bin
+node_modules
+tests
+yarn.lock
+package-lock.json
+obj
+command-output
+go.sum
+go.mod

pkg/codegen/internal/test/testdata/dash-named-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/dash-named-schema/go/foo/submodule1/init.go

@@ -6,8 +6,8 @@ package submodule1
 import (
 	"fmt"
 
+	"dash-named-schema/foo"
 	"github.com/blang/semver"
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/dash-named-schema/go/foo"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 

pkg/codegen/internal/test/testdata/dash-named-schema/go/foo/submodule1/moduleResource.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"reflect"
 
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/dash-named-schema/go/foo"
+	"dash-named-schema/foo"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 

pkg/codegen/internal/test/testdata/dash-named-schema/schema.json

@@ -44,7 +44,7 @@
       }
     },
     "go": {
-      "importBasePath": "github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/dash-named-schema/go/foo"
+      "importBasePath": "dash-named-schema/foo"
     },
     "nodejs": {
       "dependencies": {

pkg/codegen/internal/test/testdata/external-resource-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/input-collision/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/nested-module-thirdparty/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/nested-module-thirdparty/go/foo/deeply/nested/module/init.go

@@ -7,8 +7,8 @@ import (
 	"fmt"
 
 	"github.com/blang/semver"
-	"github.com/pulumi/pulumi-foo-bar/sdk/v2/go/foo-bar"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"nested-module-thirdparty/foo"
 )
 
 type module struct {
@@ -32,7 +32,7 @@ func (m *module) Construct(ctx *pulumi.Context, name, typ, urn string) (r pulumi
 }
 
 func init() {
-	version, err := foo - bar.PkgVersion()
+	version, err := foo.PkgVersion()
 	if err != nil {
 		fmt.Printf("failed to determine package version. defaulting to v1: %v\n", err)
 	}

pkg/codegen/internal/test/testdata/nested-module-thirdparty/schema.json

@@ -29,6 +29,9 @@
       "packageReferences": {
         "Pulumi": "3.12"
       }
+    },
+    "go": {
+      "importBasePath": "nested-module-thirdparty/foo"
     }
   }
 }

pkg/codegen/internal/test/testdata/nested-module/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/nested-module/go/foo/nested/module/init.go

@@ -7,8 +7,8 @@ import (
 	"fmt"
 
 	"github.com/blang/semver"
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/nested-module/go/foo"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"nested-module/foo"
 )
 
 type module struct {

pkg/codegen/internal/test/testdata/nested-module/schema.json

@@ -25,7 +25,7 @@
       }
     },
     "go": {
-      "importBasePath": "github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/nested-module/go/foo"
+      "importBasePath": "nested-module/foo"
     },
     "nodejs": {},
     "python": {}

pkg/codegen/internal/test/testdata/plain-schema-gh6957/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/provider-config-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/replace-on-change/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/resource-args-python/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/resource-args-python/go/tests/go_test.go

@@ -1,10 +1,10 @@
-package main
+package tests
 
 import (
 	"reflect"
 	"testing"
 
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/resource-args-python/go/example"
+	"resource-args-python/example"
 
 	"github.com/stretchr/testify/assert"
 )

pkg/codegen/internal/test/testdata/resource-property-overlap/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-enum-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-enum-schema/go/plant/tree/v1/init.go

@@ -7,8 +7,8 @@ import (
 	"fmt"
 
 	"github.com/blang/semver"
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"simple-enum-schema/plant"
 )
 
 type module struct {

pkg/codegen/internal/test/testdata/simple-enum-schema/go/plant/tree/v1/rubberTree.go

@@ -8,8 +8,8 @@ import (
 	"reflect"
 
 	"github.com/pkg/errors"
-	"github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"simple-enum-schema/plant"
 )
 
 type RubberTree struct {

pkg/codegen/internal/test/testdata/simple-enum-schema/go/tests/go_test.go

@@ -0,0 +1,186 @@
+package tests
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"simple-enum-schema/plant"
+	tree "simple-enum-schema/plant/tree/v1"
+)
+
+func TestEnumUsage(t *testing.T) {
+	t.Run("Success", func(t *testing.T) {
+		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
+			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
+				Container: &plant.ContainerArgs{
+					Color:    plant.ContainerColorRed,
+					Material: pulumi.String("ceramic"),
+					Size:     plant.ContainerSizeFourInch,
+				},
+				Farm: tree.Farm_Plants_R_Us,
+				Type: tree.RubberTreeVarietyRuby,
+			})
+			require.NoError(t, err)
+			require.NotNil(t, rubberTree)
+			var wg sync.WaitGroup
+			wg.Add(1)
+			pulumi.All(
+				rubberTree.URN(),
+				rubberTree.Container.Material(),
+				rubberTree.Container.Color(),
+				rubberTree.Container.Size(),
+				rubberTree.Container.Brightness(),
+				rubberTree.Type,
+			).ApplyT(func(all []interface{}) error {
+				urn := all[0].(pulumi.URN)
+				material := all[1].(*string)
+				color := all[2].(*string)
+				size := all[3].(*plant.ContainerSize)
+				brightness := all[4].(*plant.ContainerBrightness)
+				typ := all[5].(tree.RubberTreeVariety)
+				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
+				assert.Equal(t, *color, "red", "unexpected color on resource: %v", urn)
+				assert.Equal(t, *size, plant.ContainerSizeFourInch, "unexpected size on resource: %v", urn)
+				assert.Nil(t, brightness)
+				assert.Equal(t, typ, tree.RubberTreeVarietyRuby, "unexpected type on resource: %v", urn)
+				wg.Done()
+				return nil
+			})
+			wg.Wait()
+			return nil
+		}, pulumi.WithMocks("project", "stack", mocks(0))))
+	})
+
+	t.Run("StringsForRelaxedEnum", func(t *testing.T) {
+		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
+			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
+				Container: plant.ContainerArgs{
+					Color:    pulumi.String("Magenta"),
+					Material: pulumi.String("ceramic"),
+					Size:     plant.ContainerSize(22),
+				},
+				Farm: tree.Farm_Plants_R_Us,
+				Type: tree.RubberTreeVarietyRuby,
+			})
+			require.NoError(t, err)
+			require.NotNil(t, rubberTree)
+			var wg sync.WaitGroup
+			wg.Add(1)
+			pulumi.All(
+				rubberTree.URN(),
+				rubberTree.Container.Material(),
+				rubberTree.Container.Color(),
+				rubberTree.Container.Size(),
+				rubberTree.Type,
+			).ApplyT(func(all []interface{}) error {
+				urn := all[0].(pulumi.URN)
+				material := all[1].(*string)
+				color := all[2].(*string)
+				size := all[3].(*plant.ContainerSize)
+				typ := all[4].(tree.RubberTreeVariety)
+				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
+				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
+				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
+				assert.Equal(t, typ, tree.RubberTreeVarietyRuby, "unexpected type on resource: %v", urn)
+				wg.Done()
+				return nil
+			})
+			wg.Wait()
+			return nil
+		}, pulumi.WithMocks("project", "stack", mocks(1))))
+	})
+
+	t.Run("StringsForStrictEnum", func(t *testing.T) {
+		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
+			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
+				Container: plant.ContainerArgs{
+					Color:    pulumi.String("Magenta"),
+					Material: pulumi.String("ceramic"),
+					Size:     plant.ContainerSize(22),
+				},
+				Farm: tree.Farm_Plants_R_Us,
+				Type: tree.RubberTreeVarietyBurgundy,
+			})
+			require.NoError(t, err)
+			require.NotNil(t, rubberTree)
+			var wg sync.WaitGroup
+			wg.Add(1)
+			pulumi.All(
+				rubberTree.URN(),
+				rubberTree.Container.Material(),
+				rubberTree.Container.Color(),
+				rubberTree.Container.Size(),
+				rubberTree.Type,
+			).ApplyT(func(all []interface{}) error {
+				urn := all[0].(pulumi.URN)
+				material := all[1].(*string)
+				color := all[2].(*string)
+				size := all[3].(*plant.ContainerSize)
+				typ := all[4].(tree.RubberTreeVariety)
+				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
+				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
+				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
+				assert.Equal(t, typ, tree.RubberTreeVarietyBurgundy, "unexpected type on resource: %v", urn)
+				wg.Done()
+				return nil
+			})
+			wg.Wait()
+			return nil
+		}, pulumi.WithMocks("project", "stack", mocks(1))))
+	})
+
+	t.Run("EnumOutputs", func(t *testing.T) {
+		require.NoError(t, pulumi.RunErr(func(ctx *pulumi.Context) error {
+			rubberTree, err := tree.NewRubberTree(ctx, "blah", &tree.RubberTreeArgs{
+				Container: plant.ContainerArgs{
+					Color:    plant.ContainerColor("Magenta").ToContainerColorOutput().ToStringOutput(),
+					Material: pulumi.String("ceramic").ToStringOutput(),
+					Size:     plant.ContainerSize(22).ToContainerSizeOutput(),
+				},
+				Farm: tree.Farm_Plants_R_Us.ToFarmPtrOutput().ToStringPtrOutput(),
+				Type: tree.RubberTreeVarietyBurgundy.ToRubberTreeVarietyOutput(),
+			})
+			require.NoError(t, err)
+			require.NotNil(t, rubberTree)
+			var wg sync.WaitGroup
+			wg.Add(1)
+			pulumi.All(
+				rubberTree.URN(),
+				rubberTree.Container.Material(),
+				rubberTree.Container.Color(),
+				rubberTree.Container.Size(),
+				rubberTree.Type,
+			).ApplyT(func(all []interface{}) error {
+				urn := all[0].(pulumi.URN)
+				material := all[1].(*string)
+				color := all[2].(*string)
+				size := all[3].(*plant.ContainerSize)
+				typ := all[4].(tree.RubberTreeVariety)
+				assert.Equal(t, *material, "ceramic", "unexpected material on resource: %v", urn)
+				assert.Equal(t, *color, "Magenta", "unexpected color on resource: %v", urn)
+				assert.Equal(t, *size, plant.ContainerSize(22), "unexpected size on resource: %v", urn)
+				assert.Equal(t, typ, tree.RubberTreeVarietyBurgundy, "unexpected type on resource: %v", urn)
+				wg.Done()
+				return nil
+			})
+			wg.Wait()
+			return nil
+		}, pulumi.WithMocks("project", "stack", mocks(1))))
+	})
+}
+
+type mocks int
+
+func (mocks) NewResource(args pulumi.MockResourceArgs) (string, resource.PropertyMap, error) {
+	return args.Name + "_id", args.Inputs, nil
+}
+
+func (mocks) Call(args pulumi.MockCallArgs) (resource.PropertyMap, error) {
+	return args.Args, nil
+}

pkg/codegen/internal/test/testdata/simple-enum-schema/schema.json

@@ -254,10 +254,7 @@
       }
     },
     "go": {
-      "importBasePath": "github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant",
-      "packageImportAliases": {
-        "github.com/pulumi/pulumi/pkg/v3/codegen/internal/test/testdata/simple-enum-schema/go/plant/tree/v1": "treev1"
-      }
+      "importBasePath": "simple-enum-schema/plant"
     },
     "nodejs": {
       "dependencies": {

pkg/codegen/internal/test/testdata/simple-methods-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-plain-schema-with-root-package/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-plain-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-resource-schema-custom-pypackage-name/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-resource-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/internal/test/testdata/simple-yaml-schema/dotnet/version.txt

@@ -0,0 +1 @@
+0.0.0

pkg/codegen/nodejs/gen_test.go

@@ -3,9 +3,7 @@ package nodejs
 
 import (
 	"bytes"
-	"io/ioutil"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -17,15 +15,17 @@ import (
 )
 
 func TestGeneratePackage(t *testing.T) {
-	test.TestSDKCodegen(t, "nodejs", GeneratePackage, typeCheckGeneratedPackage)
+	test.TestSDKCodegen(t, &test.SDKCodegenOptions{
+		Language:   "nodejs",
+		GenPackage: GeneratePackage,
+		Checks: map[string]test.CodegenCheck{
+			"nodejs/compile": typeCheckGeneratedPackage,
+		},
+	})
 }
 
 func typeCheckGeneratedPackage(t *testing.T, pwd string) {
 	var err error
-	var npm string
-	npm, err = executable.FindExecutable("npm")
-	require.NoError(t, err)
-
 	var stdout, stderr bytes.Buffer
 	cmdOptions := integration.ProgramTestOptions{
 		Verbose: true,
@@ -33,18 +33,39 @@ func typeCheckGeneratedPackage(t *testing.T, pwd string) {
 		Stdout:  &stdout,
 	}
 
-	// TODO remove when https://github.com/pulumi/pulumi/pull/7938 lands
-	file := filepath.Join(pwd, "package.json")
-	oldFile, err := ioutil.ReadFile(file)
-	require.NoError(t, err)
-	newFile := strings.ReplaceAll(string(oldFile), "${VERSION}", "0.0.1")
-	err = ioutil.WriteFile(file, []byte(newFile), 0600)
+	// TODO: previous attempt used npm. It may be more popular and
+	// better target than yarn, however our build uses yarn in
+	// other places at the moment, and yarn does not run into the
+	// ${VERSION} problem; use yarn for now.
+	//
+	// var npm string
+	// npm, err = executable.FindExecutable("npm")
+	// require.NoError(t, err)
+	// // TODO remove when https://github.com/pulumi/pulumi/pull/7938 lands
+	// file := filepath.Join(pwd, "package.json")
+	// oldFile, err := ioutil.ReadFile(file)
+	// require.NoError(t, err)
+	// newFile := strings.ReplaceAll(string(oldFile), "${VERSION}", "0.0.1")
+	// err = ioutil.WriteFile(file, []byte(newFile), 0600)
+	// require.NoError(t, err)
+	// err = integration.RunCommand(t, "npm install", []string{npm, "i"}, pwd, &cmdOptions)
+	// require.NoError(t, err)
+
+	var yarn string
+	yarn, err = executable.FindExecutable("yarn")
 	require.NoError(t, err)
 
-	err = integration.RunCommand(t, "npm install", []string{npm, "i"}, pwd, &cmdOptions)
+	err = integration.RunCommand(t, "yarn link @pulumi/pulumi",
+		[]string{yarn, "link", "@pulumi/pulumi"}, pwd, &cmdOptions)
 	require.NoError(t, err)
-	err = integration.RunCommand(t, "typecheck ts",
+
+	err = integration.RunCommand(t, "yarn install",
+		[]string{yarn, "install"}, pwd, &cmdOptions)
+	require.NoError(t, err)
+
+	err = integration.RunCommand(t, "tsc --noEmit",
 		[]string{filepath.Join(".", "node_modules", ".bin", "tsc"), "--noEmit"}, pwd, &cmdOptions)
+
 	if err != nil {
 		stderr := stderr.String()
 		if len(stderr) > 0 {

pkg/codegen/python/gen_test.go

@@ -59,21 +59,27 @@ func TestRelPathToRelImport(t *testing.T) {
 }
 
 func TestGeneratePackage(t *testing.T) {
-	test.TestSDKCodegen(t, "python", GeneratePackage, typeCheckGeneratedPackage)
+	test.TestSDKCodegen(t, &test.SDKCodegenOptions{
+		Language:   "python",
+		GenPackage: GeneratePackage,
+		Checks: map[string]test.CodegenCheck{
+			"python/py_compile": pyCompileCheck,
+		},
+	})
 }
 
-// We can't type check a python program. We just check for syntax errors.
-func typeCheckGeneratedPackage(t *testing.T, pwd string) {
+// Checks generated code for syntax errors with `python -m compile`.
+func pyCompileCheck(t *testing.T, codeDir string) {
 	ex, _, err := python.CommandPath()
 	require.NoError(t, err)
 	cmdOptions := integration.ProgramTestOptions{}
-	err = filepath.Walk(pwd, func(path string, info filesystem.FileInfo, err error) error {
+	err = filepath.Walk(codeDir, func(path string, info filesystem.FileInfo, err error) error {
 		require.NoError(t, err) // an error in the walk
 		if info.Mode().IsRegular() && strings.HasSuffix(info.Name(), ".py") {
 			path, err = filepath.Abs(path)
 			require.NoError(t, err)
 			err = integration.RunCommand(t, "python syntax check",
-				[]string{ex, "-m", "py_compile", path}, pwd, &cmdOptions)
+				[]string{ex, "-m", "py_compile", path}, codeDir, &cmdOptions)
 			require.NoError(t, err)
 		}
 		return nil

pkg/codegen/schema/pulumi.json

@@ -0,0 +1,493 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://github.com/pulumi/pulumi/blob/master/pkg/codegen/schema.json",
+    "title": "Pulumi Package Metaschema",
+    "description": "A description of the schema for a Pulumi Package",
+    "type": "object",
+    "properties": {
+        "name": {
+            "description": "The unqualified name of the package (e.g. \"aws\", \"azure\", \"gcp\", \"kubernetes\", \"random\")",
+            "type": "string",
+            "pattern": "^[^0-9][-a-zA-Z0-9]*$"
+        },
+        "version": {
+            "description": "The version of the package. The version must be valid semver.",
+            "type": "string",
+            "pattern": "^v?(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)\\.(?P<patch>0|[1-9]\\d*)(?:-(?P<prerelease>(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
+        },
+        "description": {
+            "description": "The description of the package. Descriptions are interpreted as Markdown.",
+            "type": "string"
+        },
+        "keywords": {
+            "description": "The list of keywords that are associated with the package, if any.",
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        },
+        "homepage": {
+            "description": "The package's homepage.",
+            "type": "string"
+        },
+        "license": {
+            "description": "The name of the license used for the package's contents.",
+            "type": "string"
+        },
+        "attribution": {
+            "description": "Freeform text attribution of derived work, if required.",
+            "type": "string"
+        },
+        "repository": {
+            "description": "The URL at which the package's sources can be found.",
+            "type": "string"
+        },
+        "logoUrl": {
+            "description": "The URL of the package's logo, if any.",
+            "type": "string"
+        },
+        "pluginDownloadUrl": {
+            "description": "The URL to use when downloading the provider plugin binary.",
+            "type": "string"
+        },
+        "meta": {
+            "description": "Format metadata about this package.",
+            "type": "object",
+            "properties": {
+                "moduleFormat": {
+                    "description": "A regex that is used by the importer to extract a module name from the module portion of a type token. Packages that use the module format \"namespace1/namespace2/.../namespaceN\" do not need to specify a format. The regex must define one capturing group that contains the module name, which must be formatted as \"namespace1/namespace2/...namespaceN\".",
+                    "type": "string",
+                    "format": "regex"
+                }
+            },
+            "required": ["moduleFormat"]
+        },
+        "config": {
+            "description": "The package's configuration variables.",
+            "type": "object",
+            "properties": {
+                "variables": {
+                    "description": "A map from variable name to propertySpec that describes a package's configuration variables.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/$defs/propertySpec"
+                    }
+                },
+                "required": {
+                    "description": "A list of the names of the package's required configuration variables.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "types": {
+            "description": "A map from type token to complexTypeSpec that describes the set of complex types (i.e. object, enum) defined by this package.",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "#/$defs/complexTypeSpec"
+            },
+            "propertyNames": {
+                "$ref": "#/$defs/token"
+            }
+        },
+        "provider": {
+            "description": "The provider type for this package.",
+            "$ref": "#/$defs/resourceSpec"
+        },
+        "resources": {
+            "description": "A map from type token to resourceSpec that describes the set of resources and components defined by this package.",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "#/$defs/resourceSpec"
+            },
+            "propertyNames": {
+                "$ref": "#/$defs/token"
+            }
+        },
+        "functions": {
+            "description": "A map from token to functionSpec that describes the set of functions defined by this package.",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "#/$defs/functionSpec"
+            },
+            "propertyNames": {
+                "$ref": "#/$defs/token"
+            }
+        },
+        "language": {
+            "description": "Additional language-specific data about the package.",
+            "type": "object"
+        }
+    },
+    "additionalProperties": false,
+    "required": [
+        "name"
+    ],
+    "$defs": {
+        "token": {
+            "title": "Token",
+            "type": "string",
+            "$comment": "In the regex below, the 'module' portion of the token is optional. However, a missing module component creates a '::', which breaks URNs ('::' is the URN delimiter). We have many test schemas that use an empty module component successfully, as they never create URNs; while these are _probably_ the only places that need updating, it might be possible that there are module-less type tokens in the wild elsewhere and we may need to remain compatible with those tokens.",
+            "pattern": "^[^0-9][-a-zA-Z0-9]*:([^0-9][a-zA-Z0-9._/]*)?:[^0-9][a-zA-Z0-9._/]*$"
+        },
+        "typeSpec": {
+            "title": "Type Reference",
+            "description": "A reference to a type",
+            "type": "object",
+            "properties": {
+                "plain": {
+                    "description": "Indicates that when used as an input, this type does not accept eventual values.",
+                    "type": "boolean"
+                }
+            },
+            "oneOf": [
+                {
+                    "title": "Primitive Type",
+                    "description": "A reference to a primitive type.",
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "description": "The primitive type, if any",
+                            "type": "string",
+                            "enum": ["boolean", "integer", "number", "string"]
+                        }
+                    },
+                    "required": ["type"]
+                },
+                {
+                    "title": "Array Type",
+                    "description": "A reference to an array type.",
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "const": "array"
+                        },
+                        "items": {
+                            "description": "The element type of an array",
+                            "$ref": "#/$defs/typeSpec"
+                        }
+                    },
+                    "required": ["type", "items"]
+                },
+                {
+                    "title": "Map Type",
+                    "description": "A reference to a map type.",
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "const": "object"
+                        },
+                        "additionalProperties": {
+                            "description": "The element type of the map. Defaults to \"string\" when omitted.",
+                            "$ref": "#/$defs/typeSpec"
+                        }
+                    },
+                    "required": ["type"]
+                },
+                {
+                    "title": "Named Type",
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "description": "ignored; present for compatibility with existing schemas",
+                            "type": "string"
+                        },
+                        "$ref": {
+                            "description": "A reference to a type in this or another document. For example, the built-in Archive, Asset, and Any\ntypes are referenced as \"pulumi.json#/Archive\", \"pulumi.json#/Asset\", and \"pulumi.json#/Any\", respectively.\nA type from this document is referenced as \"#/types/pulumi:type:token\".\nA type from another document is referenced as \"path#/types/pulumi:type:token\", where path is of the form:\n  \"/provider/vX.Y.Z/schema.json\" or \"pulumi.json\" or \"http[s]://example.com/provider/vX.Y.Z/schema.json\"\nA resource from this document is referenced as \"#/resources/pulumi:type:token\".\nA resource from another document is referenced as \"path#/resources/pulumi:type:token\", where path is of the form:\n  \"/provider/vX.Y.Z/schema.json\" or \"pulumi.json\" or \"http[s]://example.com/provider/vX.Y.Z/schema.json\"",
+                            "type": "string",
+                            "format": "uri-reference"
+                        }
+                    },
+                    "required": ["$ref"]
+                },
+                {
+                    "title": "Union Type",
+                    "description": "A reference to a union type.",
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "description": "The underlying primitive type of the union, if any",
+                            "type": "string",
+                            "enum": ["boolean", "integer", "number", "string"]
+                        },
+                        "oneOf": {
+                            "description": "If present, indicates that values of the type may be one of any of the listed types",
+                            "type": "array",
+                            "items": {
+                                "$ref": "#/$defs/typeSpec"
+                            },
+                            "minItems": 2
+                        },
+                        "discriminator": {
+                            "description": "Informs the consumer of an alternative schema based on the value associated with it",
+                            "type": "object",
+                            "properties": {
+                                "propertyName": {
+                                    "description": "PropertyName is the name of the property in the payload that will hold the discriminator value",
+                                    "type": "string",
+                                    "minLength": 1
+                                },
+                                "mapping": {
+                                    "description": "an optional object to hold mappings between payload values and schema names or references",
+                                    "type": "object",
+                                    "additionalProperties": {
+                                        "type": "string"
+                                    }
+                                }
+                            },
+                            "required": [
+                                "propertyName"
+                            ]
+                        }
+                    },
+                    "required": ["oneOf"]
+                }
+            ]
+        },
+        "propertySpec": {
+            "title": "Property Definition",
+            "description": "Describes an object or resource property",
+            "type": "object",
+            "allOf": [
+                { "$ref": "#/$defs/typeSpec" }
+            ],
+            "properties": {
+                "description": {
+                    "description": "The description of the property, if any. Interpreted as Markdown.",
+                    "type": "string"
+                },
+                "const": {
+                    "description": "The constant value for the property, if any. The type of the value must be assignable to the type of the property.",
+                    "type": ["boolean", "number", "string"]
+                },
+                "default": {
+                    "description": "The default value for the property, if any. The type of the value must be assignable to the type of the property.",
+                    "type": ["boolean", "number", "string"]
+                },
+                "defaultInfo": {
+                    "description": "Additional information about the property's default value, if any.",
+                    "type": "object",
+                    "properties": {
+                        "environment": {
+                            "description": "A set of environment variables to probe for a default value.",
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "language": {
+                            "description": "Additional language-specific data about the default value.",
+                            "type": "object"
+                        }
+                    },
+                    "required": ["environment"]
+                },
+                "deprecationMessage": {
+                    "description": "Indicates whether or not the property is deprecated",
+                    "type": "string"
+                },
+                "language": {
+                    "description": "Additional language-specific data about the property.",
+                    "type": "object"
+                },
+                "secret": {
+                    "description": "Specifies whether the property is secret (default false).",
+                    "type": "boolean"
+                },
+                "replaceOnChanges": {
+                    "description": "Specifies whether a change to the property causes its containing resource to be replaced instead of updated (default false).",
+                    "type": "boolean"
+                }
+            }
+        },
+        "complexTypeSpec": {
+            "title": "Type Definition",
+            "description": "Describes an object or enum type.",
+            "type": "object",
+            "properties": {
+                "description": {
+                    "description": "The description of the type, if any. Interpreted as Markdown.",
+                    "type": "string"
+                },
+                "language": {
+                    "description": "Additional language-specific data about the type.",
+                    "type": "object"
+                }
+            },
+            "oneOf": [
+                {
+                    "title": "Object Type Definition",
+                    "type": "object",
+                    "allOf": [
+                        { "$ref": "#/$defs/objectTypeSpec" }
+                    ],
+                    "properties": {
+                        "type": {
+                            "const": "object"
+                        }
+                    }
+                },
+                { "$ref": "#/$defs/enumTypeSpec" }
+            ]
+        },
+        "objectTypeSpec": {
+            "title": "Object Type Details",
+            "description": "Describes an object type",
+            "type": "object",
+            "properties": {
+                "properties": {
+                    "description": "A map from property name to propertySpec that describes the object's properties.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/$defs/propertySpec"
+                    }
+                },
+                "required": {
+                    "description": "A list of the names of an object type's required properties. These properties must be set for inputs and will always be set for outputs.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "enumTypeSpec": {
+            "title": "Enum Type Definition",
+            "description": "Describes an enum type",
+            "type": "object",
+            "properties" :{
+                "type": {
+                    "description": "The underlying primitive type of the enum",
+                    "type": "string",
+                    "enum": ["boolean", "integer", "number", "string"]
+                },
+                "enum": {
+                    "description": "The list of possible values for the enum",
+                    "type": "array",
+                    "items": {
+                        "title": "Enum Value Definition",
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "description": "If present, overrides the name of the enum value that would usually be derived from the value.",
+                                "type": "string"
+                            },
+                            "description": {
+                                "description": "The description of the enum value, if any. Interpreted as Markdown.",
+                                "type": "string"
+                            },
+                            "value": {
+                                "description": "The enum value itself",
+                                "type": ["boolean", "integer", "number", "string"]
+                            },
+                            "deprecationMessage": {
+                                "description": "Indicates whether or not the value is deprecated.",
+                                "type": "string"
+                            }
+                        },
+                        "required": ["value"]
+                    }
+                }
+            },
+            "required": ["type", "enum"]
+        },
+        "resourceSpec": {
+            "title": "Resource Definition",
+            "description": "Describes a resource or component.",
+            "type": "object",
+            "allOf": [
+                { "$ref": "#/$defs/objectTypeSpec" }
+            ],
+            "properties": {
+                "description": {
+                    "description": "The description of the resource, if any. Interpreted as Markdown.",
+                    "type": "string"
+                },
+                "inputProperties": {
+                    "description": "A map from property name to propertySpec that describes the resource's input properties.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "$ref": "#/$defs/propertySpec"
+                    }
+                },
+                "requiredInputs": {
+                    "description": "A list of the names of the resource's required input properties.",
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "stateInputs": {
+                    "description": "An optional objectTypeSpec that describes additional inputs that mau be necessary to get an existing resource. If this is unset, only an ID is necessary.",
+                    "$ref": "#/$defs/objectTypeSpec"
+                },
+                "aliases": {
+                    "description": "The list of aliases for the resource.",
+                    "type": "array",
+                    "items": {
+                        "title": "Alias Definition",
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "description": "The name portion of the alias, if any",
+                                "type": "string"
+                            },
+                            "project": {
+                                "description": "The project portion of the alias, if any",
+                                "type": "string"
+                            },
+                            "type": {
+                                "description": "The type portion of the alias, if any",
+                                "type": "string"
+                            }
+                        }
+                    }
+                },
+                "deprecationMessage": {
+                    "description": "Indicates whether or not the resource is deprecated",
+                    "type": "string"
+                },
+                "isComponent": {
+                    "description": "Indicates whether or not the resource is a component.",
+                    "type": "boolean"
+                },
+                "methods": {
+                    "description": "A map from method name to function token that describes the resource's method set.",
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
+        "functionSpec": {
+            "title": "Function Definition",
+            "description": "Describes a function.",
+            "type": "object",
+            "properties": {
+                "description": {
+                    "description": "The description of the function, if any. Interpreted as Markdown.",
+                    "type": "string"
+                },
+                "inputs": {
+                    "description": "The bag of input values for the function, if any.",
+                    "$ref": "#/$defs/objectTypeSpec"
+                },
+                "outputs": {
+                    "description": "The bag of output values for the function, if any.",
+                    "$ref": "#/$defs/objectTypeSpec"
+                },
+                "deprecationMessage": {
+                    "description": "Indicates whether or not the function is deprecated",
+                    "type": "string"
+                },
+                "language": {
+                    "description": "Additional language-specific data about the function.",
+                    "type": "object"
+                }
+            }
+        }
+    }
+}

pkg/testing/integration/command.go

@@ -83,9 +83,15 @@ func RunCommand(t *testing.T, name string, args []string, wd string, opts *Progr
 		t.Logf("Invoke '%v' failed: %s\n", command, cmdutil.DetailedError(runerr))
 
 		if !opts.Verbose {
+			stderr := opts.Stderr
+
+			if stderr == nil {
+				stderr = os.Stderr
+			}
+
 			// Make sure we write the output in case of a failure to stderr so
 			// tests can assert the shape of the error message.
-			_, _ = fmt.Fprintf(opts.Stderr, "%s\n", string(runout))
+			_, _ = fmt.Fprintf(stderr, "%s\n", string(runout))
 		}
 	}