diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 33819dfae..5f7595d36 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -209,7 +209,7 @@ jobs: node-version: [ 14.x ] # See scripts/tests_subsets.py when editing - test-subset: [ integration, auto-and-lifecycletest, native, etc ] + test-subset: [ integration, integration-and-codegen, auto, etc ] runs-on: ${{ matrix.platform }} steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0306b94cb..3675fcfaf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -300,7 +300,7 @@ jobs: node-version: [ 14.x ] # See scripts/tests_subsets.py when editing - test-subset: [ integration, auto-and-lifecycletest, native, etc ] + test-subset: [ integration, integration-and-codegen, auto, etc ] runs-on: ${{ matrix.platform }} steps: diff --git a/.github/workflows/run-build-and-acceptance-tests.yml b/.github/workflows/run-build-and-acceptance-tests.yml index 4ba799ecd..58cb27d13 100644 --- a/.github/workflows/run-build-and-acceptance-tests.yml +++ b/.github/workflows/run-build-and-acceptance-tests.yml @@ -119,13 +119,29 @@ jobs: strategy: matrix: platform: [ ubuntu-latest, macos-latest ] - go-version: [1.16.x] + go-version: [ 1.16.x ] python-version: [ 3.9.x ] dotnet-version: [ 3.1.x ] node-version: [ 14.x ] # See scripts/tests_subsets.py when editing - test-subset: [ integration, auto-and-lifecycletest, native, etc ] + test-subset: [ integration, integration-and-codegen, auto, etc ] + + exclude: + - platform: macos-latest + + # Only run catch-all `etc` test-subset on Mac for PR + # verification because of a throughput bottleneck on Mac + # runners. Note that `master.yml` specifies all test subsets + # to still run on `master` branch verifications. + include: + - platform: macos-latest + test-subset: etc + go-version: 1.16.x + python-version: 3.9.x + dotnet-version: 3.1.x + node-version: 14.x + fail-fast: false if: github.event_name == 'repository_dispatch' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ${{ matrix.platform }} diff --git a/scripts/test_subsets.py b/scripts/test_subsets.py index 42b8dba46..a092ba685 100644 --- a/scripts/test_subsets.py +++ b/scripts/test_subsets.py @@ -14,25 +14,63 @@ A note on the format of TEST_SUBSETS. The keys are test configuration names, and the values are lists of either Go packages containing the tests, or test suites names as passed to `run-testsuite.py`. +The special `etc` test subset catches all unlisted tests. This subset +will always run on every PR. Other tests subsets may be skipped on PR +verification for platforms such as Mac OS where there is currently a +shortage of runners; these tests will still run on `master` and +`release` verifications. + """ TEST_SUBSETS = { 'integration': [ - 'github.com/pulumi/pulumi/tests/integration' + 'github.com/pulumi/pulumi/tests/integration', ], - 'auto-and-lifecycletest': [ - 'auto-dotnet', - 'auto-nodejs', - 'auto-python', + + 'integration-and-codegen': [ + 'github.com/pulumi/pulumi/tests/integration/aliases', + 'github.com/pulumi/pulumi/tests/integration/custom_timeouts', + 'github.com/pulumi/pulumi/tests/integration/delete_before_create', + 'github.com/pulumi/pulumi/tests/integration/dependency_steps', + 'github.com/pulumi/pulumi/tests/integration/double_pending_delete', + 'github.com/pulumi/pulumi/tests/integration/duplicate_urns', + 'github.com/pulumi/pulumi/tests/integration/partial_state', + 'github.com/pulumi/pulumi/tests/integration/policy', + 'github.com/pulumi/pulumi/tests/integration/protect_resources', + 'github.com/pulumi/pulumi/tests/integration/query', + 'github.com/pulumi/pulumi/tests/integration/read/import_acquire', + 'github.com/pulumi/pulumi/tests/integration/read/read_dbr', + 'github.com/pulumi/pulumi/tests/integration/read/read_relinquish', + 'github.com/pulumi/pulumi/tests/integration/read/read_replace', + 'github.com/pulumi/pulumi/tests/integration/recreate_resource_check', + 'github.com/pulumi/pulumi/tests/integration/steps', + 'github.com/pulumi/pulumi/tests/integration/targets', + 'github.com/pulumi/pulumi/tests/integration/transformations', + 'github.com/pulumi/pulumi/tests/integration/types', + 'github.com/pulumi/pulumi/pkg/v3/codegen', + 'github.com/pulumi/pulumi/pkg/v3/codegen/docs', + 'github.com/pulumi/pulumi/pkg/v3/codegen/dotnet', + 'github.com/pulumi/pulumi/pkg/v3/codegen/go', + 'github.com/pulumi/pulumi/pkg/v3/codegen/hcl2', + 'github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/model', + 'github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax', + 'github.com/pulumi/pulumi/pkg/v3/codegen/importer', + 'github.com/pulumi/pulumi/pkg/v3/codegen/nodejs', + 'github.com/pulumi/pulumi/pkg/v3/codegen/python', + 'github.com/pulumi/pulumi/pkg/v3/codegen/schema', + ], + + 'auto': [ + # Primary Go-driven Auto API tests 'github.com/pulumi/pulumi/sdk/v3/go/auto', - 'github.com/pulumi/pulumi/pkg/v3/engine/lifeycletest' - ], - 'native': [ - 'dotnet-test', - 'istanbul', - 'istanbul-with-mocks', - 'python/lib/test', - 'python/lib/test/langhost/resource_thens', - 'python/lib/test_with_mocks' + + # Auto API tests driven by dotnet + 'auto-dotnet', + + # Auto API tests driven by npm + 'auto-nodejs', + + # Auto API tests driven by pytest + 'auto-python', ] }