diff --git a/.travis.yml b/.travis.yml index 188019fbe..2fd3d0c9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=$PATH:$HOME/Library/Python/2.7/bin; export PIP=pip2.7; fi # Install the AWS CLI so that we can publish the resulting release (if applicable) at the end. - $PIP install --upgrade --user awscli + - export PULUMI_FAILED_TESTS_DIR=$(mktemp -d); echo "${PULUMI_FAILED_TESTS_DIR}" cache: yarn: true install: @@ -46,5 +47,8 @@ before_script: - python -c 'import fcntl, os, sys; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); print("stdout was " + ("nonblocking" if flags & os.O_NONBLOCK else "blocking")); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)' script: - make travis_${TRAVIS_EVENT_TYPE} TEST_FAST_TIMEOUT=10m +after_failure: + # Copy any data from failed tests to S3. + - aws --region us-west-2 s3 cp --recursive "${PULUMI_FAILED_TESTS_DIR}" "s3://eng.pulumi.com/travis-logs/${TRAVIS_REPO_SLUG}/${TRAVIS_JOB_NUMBER}/failures" notifications: webhooks: https://ufci1w66n3.execute-api.us-west-2.amazonaws.com/stage/travis diff --git a/pkg/backend/cloud/backend.go b/pkg/backend/cloud/backend.go index c8a63214e..133aa4b18 100644 --- a/pkg/backend/cloud/backend.go +++ b/pkg/backend/cloud/backend.go @@ -472,7 +472,7 @@ func (b *cloudBackend) makeProgramUpdateRequest(stackName tokens.QName) (apitype wireConfig := make(map[tokens.ModuleMember]apitype.ConfigValue) for k, cv := range cfg { v, err := cv.Value(config.NopDecrypter) - contract.Assert(err == nil) + contract.AssertNoError(err) wireConfig[k] = apitype.ConfigValue{ String: v, diff --git a/pkg/backend/local/crypto.go b/pkg/backend/local/crypto.go index 29417a410..d15c02e5b 100644 --- a/pkg/backend/local/crypto.go +++ b/pkg/backend/local/crypto.go @@ -75,7 +75,7 @@ func symmetricCrypter() (config.Crypter, error) { // Encrypt a message and store it with the salt so we can test if the password is correct later. crypter := config.NewSymmetricCrypterFromPassphrase(phrase, salt) msg, err := crypter.EncryptValue("pulumi") - contract.Assert(err == nil) + contract.AssertNoError(err) // Now store the result on the package and save it. pkg.EncryptionSalt = fmt.Sprintf("v1:%s:%s", base64.StdEncoding.EncodeToString(salt), msg) diff --git a/pkg/engine/plan.go b/pkg/engine/plan.go index 60f0308a4..c3df62e62 100644 --- a/pkg/engine/plan.go +++ b/pkg/engine/plan.go @@ -227,7 +227,7 @@ func printConfig(b *bytes.Buffer, cfg config.Map) { sort.Strings(keys) for _, key := range keys { v, err := cfg[tokens.ModuleMember(key)].Value(config.NewBlindingDecrypter()) - contract.Assert(err == nil) + contract.AssertNoError(err) b.WriteString(fmt.Sprintf(" %v: %v\n", key, v)) } } diff --git a/pkg/resource/config/crypt.go b/pkg/resource/config/crypt.go index 5671f84e5..f4e4249ae 100644 --- a/pkg/resource/config/crypt.go +++ b/pkg/resource/config/crypt.go @@ -131,10 +131,10 @@ func encryptAES256GCGM(plaintext string, key []byte) ([]byte, []byte) { contract.Assertf(err == nil, "could not read from system random source") block, err := aes.NewCipher(key) - contract.Assert(err == nil) + contract.AssertNoError(err) aesgcm, err := cipher.NewGCM(block) - contract.Assert(err == nil) + contract.AssertNoError(err) msg := aesgcm.Seal(nil, nonce, []byte(plaintext), nil) @@ -145,10 +145,10 @@ func decryptAES256GCM(ciphertext []byte, key []byte, nonce []byte) (string, erro contract.Requiref(len(key) == SymmetricCrypterKeyBytes, "key", "AES-256-GCM needs a 32 byte key") block, err := aes.NewCipher(key) - contract.Assert(err == nil) + contract.AssertNoError(err) aesgcm, err := cipher.NewGCM(block) - contract.Assert(err == nil) + contract.AssertNoError(err) msg, err := aesgcm.Open(nil, nonce, ciphertext, nil) diff --git a/pkg/resource/resource_id.go b/pkg/resource/resource_id.go index e74db443a..828a99841 100644 --- a/pkg/resource/resource_id.go +++ b/pkg/resource/resource_id.go @@ -61,7 +61,7 @@ func NewUniqueHex(prefix string, randlen, maxlen int) (string, error) { bs := make([]byte, randlen+1/2) n, err := cryptorand.Read(bs) - contract.Assert(err == nil) + contract.AssertNoError(err) contract.Assert(n == len(bs)) return prefix + hex.EncodeToString(bs)[:randlen], nil diff --git a/pkg/testing/integration/program.go b/pkg/testing/integration/program.go index 72c47e373..eafc46868 100644 --- a/pkg/testing/integration/program.go +++ b/pkg/testing/integration/program.go @@ -43,7 +43,6 @@ type RuntimeValidationStackInfo struct { type EditDir struct { Dir string ExtraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo) - Additive bool // set to true to keep the prior test dir, applying this edit atop. // ExpectFailure is true if we expect this test to fail. This is very coarse grained, and will essentially // tolerate *any* failure in the program (IDEA: in the future, offer a way to narrow this down more). @@ -277,17 +276,45 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt // // All commands must return success return codes for the test to succeed, unless ExpectFailure is true. func ProgramTest(t *testing.T, opts *ProgramTestOptions) { - err := TestLifeCycleInitAndDestroy(t, opts, testPreviewUpdateAndEdits) + err := testLifeCycleInitAndDestroy(t, opts, testPreviewUpdateAndEdits) assert.NoError(t, err) } -func TestLifeCycleInitAndDestroy( +func uniqueSuffix() string { + // .. + timestamp := time.Now().Format("20060102-150405") + suffix, err := resource.NewUniqueHex("."+timestamp+".", 5, -1) + contract.AssertNoError(err) + return suffix +} + +func testLifeCycleInitAndDestroy( t *testing.T, opts *ProgramTestOptions, - between func(*testing.T, *ProgramTestOptions, string) (string, error)) error { + between func(*testing.T, *ProgramTestOptions, string) error) error { t.Parallel() - dir, err := TestLifeCycleInitialize(t, opts) + dir, err := CopyTestToTemporaryDirectory(t, opts) + if err != nil { + return errors.Wrap(err, "Couldn't copy test to temporary directory") + } + + defer func() { + if t.Failed() { + // Test failed -- keep temporary files around for debugging. + // Maybe also copy to "failed tests" directory. + failedTestsDir := os.Getenv("PULUMI_FAILED_TESTS_DIR") + if failedTestsDir != "" { + dest := filepath.Join(failedTestsDir, t.Name()+uniqueSuffix()) + contract.IgnoreError(fsutil.CopyFile(dest, dir, nil)) + } + } else { + // Test passed -- delete temporary files. + contract.IgnoreError(os.RemoveAll(dir)) + } + }() + + err = testLifeCycleInitialize(t, opts, dir) if err != nil { return err } @@ -295,25 +322,17 @@ func TestLifeCycleInitAndDestroy( // Ensure that before we exit, we attempt to destroy and remove the stack. defer func() { if dir != "" { - destroyErr := TestLifeCycleDestroy(t, opts, dir) + destroyErr := testLifeCycleDestroy(t, opts, dir) assert.NoError(t, destroyErr) } }() - dir, err = between(t, opts, dir) - return err + return between(t, opts, dir) } -func TestLifeCycleInitialize(t *testing.T, opts *ProgramTestOptions) (string, error) { +func testLifeCycleInitialize(t *testing.T, opts *ProgramTestOptions, dir string) error { stackName := opts.GetStackName() - // Perform the initial stack creation. - - dir, err := CopyTestToTemporaryDirectory(t, opts) - if err != nil { - return "", err - } - // If RelativeWorkDir is specified, apply that relative to the temp folder for use as working directory during tests. if opts.RelativeWorkDir != "" { dir = path.Join(dir, opts.RelativeWorkDir) @@ -325,8 +344,8 @@ func TestLifeCycleInitialize(t *testing.T, opts *ProgramTestOptions) (string, er initArgs := []string{"init"} initArgs = addFlagIfNonNil(initArgs, "--owner", opts.Owner) initArgs = addFlagIfNonNil(initArgs, "--name", opts.Repo) - if err = RunCommand(t, "pulumi-init", opts.PulumiCmd(initArgs), dir, opts); err != nil { - return "", err + if err := RunCommand(t, "pulumi-init", opts.PulumiCmd(initArgs), dir, opts); err != nil { + return err } // Login as needed. @@ -337,13 +356,13 @@ func TestLifeCycleInitialize(t *testing.T, opts *ProgramTestOptions) (string, er // Set the "use alt location" flag so this test doesn't interact with any credentials already on the machine. // e.g. replacing the current user's with that of a test account. - if err = os.Setenv(workspace.UseAltCredentialsLocationEnvVar, "1"); err != nil { + if err := os.Setenv(workspace.UseAltCredentialsLocationEnvVar, "1"); err != nil { t.Fatalf("error setting env var '%s': %v", workspace.UseAltCredentialsLocationEnvVar, err) } args := opts.PulumiCmd([]string{"login", "--cloud-url", opts.CloudURL}) - if err = RunCommand(t, "pulumi-login", args, dir, opts); err != nil { - return "", err + if err := RunCommand(t, "pulumi-login", args, dir, opts); err != nil { + return err } } @@ -355,28 +374,28 @@ func TestLifeCycleInitialize(t *testing.T, opts *ProgramTestOptions) (string, er stackInitArgs = addFlagIfNonNil(stackInitArgs, "--cloud-url", opts.CloudURL) stackInitArgs = addFlagIfNonNil(stackInitArgs, "--ppc", opts.PPCName) } - if err = RunCommand(t, "pulumi-stack-init", opts.PulumiCmd(stackInitArgs), dir, opts); err != nil { - return "", err + if err := RunCommand(t, "pulumi-stack-init", opts.PulumiCmd(stackInitArgs), dir, opts); err != nil { + return err } for key, value := range opts.Config { - if err = RunCommand(t, "pulumi-config", + if err := RunCommand(t, "pulumi-config", opts.PulumiCmd([]string{"config", "set", key, value}), dir, opts); err != nil { - return "", err + return err } } for key, value := range opts.Secrets { - if err = RunCommand(t, "pulumi-config", + if err := RunCommand(t, "pulumi-config", opts.PulumiCmd([]string{"config", "set", "--secret", key, value}), dir, opts); err != nil { - return "", err + return err } } - return dir, nil + return nil } -func TestLifeCycleDestroy(t *testing.T, opts *ProgramTestOptions, dir string) error { +func testLifeCycleDestroy(t *testing.T, opts *ProgramTestOptions, dir string) error { stackName := opts.GetStackName() destroy := opts.PulumiCmd([]string{"destroy", "--yes"}) @@ -403,27 +422,27 @@ func TestLifeCycleDestroy(t *testing.T, opts *ProgramTestOptions, dir string) er return nil } -func testPreviewUpdateAndEdits(t *testing.T, opts *ProgramTestOptions, dir string) (string, error) { +func testPreviewUpdateAndEdits(t *testing.T, opts *ProgramTestOptions, dir string) error { // Now preview and update the real changes. pioutil.MustFprintf(opts.Stdout, "Performing primary preview and update\n") initErr := previewAndUpdate(t, opts, dir, "initial", opts.ExpectFailure) // If the initial preview/update failed, just exit without trying the rest (but make sure to destroy). if initErr != nil { - return dir, initErr + return initErr } // Perform an empty preview and update; nothing is expected to happen here. if !opts.Quick { pioutil.MustFprintf(opts.Stdout, "Performing empty preview and update (no changes expected)\n") if err := previewAndUpdate(t, opts, dir, "empty", false); err != nil { - return dir, err + return err } } // Run additional validation provided by the test options, passing in the checkpoint info. if err := performExtraRuntimeValidation(t, opts, opts.ExtraRuntimeValidation, dir); err != nil { - return dir, err + return err } // If there are any edits, apply them and run a preview and update for each one. @@ -467,25 +486,24 @@ func previewAndUpdate(t *testing.T, opts *ProgramTestOptions, dir string, name s return nil } -func testEdits(t *testing.T, opts *ProgramTestOptions, dir string) (string, error) { +func testEdits(t *testing.T, opts *ProgramTestOptions, dir string) error { for i, edit := range opts.EditDirs { var err error - if dir, err = testEdit(t, opts, dir, i, edit); err != nil { - return dir, err + if err = testEdit(t, opts, dir, i, edit); err != nil { + return err } } - return dir, nil + return nil } -func testEdit(t *testing.T, opts *ProgramTestOptions, dir string, i int, edit EditDir) (string, error) { +func testEdit(t *testing.T, opts *ProgramTestOptions, dir string, i int, edit EditDir) error { pioutil.MustFprintf(opts.Stdout, "Applying edit '%v' and rerunning preview and update\n", edit.Dir) - newDir, err := prepareProject(t, opts, edit.Dir, dir, edit.Additive) + err := prepareProject(t, opts, edit.Dir, dir) if err != nil { - return dir, errors.Wrapf(err, "Expected to apply edit %v atop %v, but got an error", edit, dir) + return errors.Wrapf(err, "Expected to apply edit %v atop %v, but got an error", edit, dir) } - dir = newDir oldStdOut := opts.Stdout oldStderr := opts.Stderr oldVerbose := opts.Verbose @@ -506,13 +524,13 @@ func testEdit(t *testing.T, opts *ProgramTestOptions, dir string, i int, edit Ed }() if err = previewAndUpdate(t, opts, dir, fmt.Sprintf("edit-%d", i), edit.ExpectFailure); err != nil { - return dir, err + return err } if err = performExtraRuntimeValidation(t, opts, edit.ExtraRuntimeValidation, dir); err != nil { - return dir, err + return err } - return dir, nil + return nil } func performExtraRuntimeValidation( @@ -584,12 +602,12 @@ func CopyTestToTemporaryDirectory(t *testing.T, opts *ProgramTestOptions) (dir s // Set up a prefix so that all output has the test directory name in it. This is important for debugging // because we run tests in parallel, and so all output will be interleaved and difficult to follow otherwise. - dir = opts.Dir + sourceDir := opts.Dir var prefix string if len(dir) <= 30 { - prefix = fmt.Sprintf("[ %30.30s ] ", dir) + prefix = fmt.Sprintf("[ %30.30s ] ", sourceDir) } else { - prefix = fmt.Sprintf("[ %30.30s ] ", dir[len(dir)-30:]) + prefix = fmt.Sprintf("[ %30.30s ] ", sourceDir[len(sourceDir)-30:]) } stdout := opts.Stdout if stdout == nil { @@ -606,14 +624,42 @@ func CopyTestToTemporaryDirectory(t *testing.T, opts *ProgramTestOptions) (dir s pioutil.MustFprintf(opts.Stdout, "pulumi: %v\n", opts.Bin) pioutil.MustFprintf(opts.Stdout, "yarn: %v\n", opts.YarnBin) - // Now copy the source project, excluding the .pulumi directory. - dir, err = prepareProject(t, opts, dir, "", false) + // Copy the source project + stackName := string(opts.GetStackName()) + targetDir, err := ioutil.TempDir("", stackName+"-") + if err != nil { + return "", errors.Wrap(err, "Couldn't create temporary directory") + } + + // Clean up the temporary directory on failure + defer func() { + if err != nil { + contract.IgnoreError(os.RemoveAll(targetDir)) + } + }() + + err = prepareProject(t, opts, sourceDir, targetDir) if err != nil { return "", errors.Wrapf(err, "Failed to copy source project %v to a new temp dir", dir) } - pioutil.MustFprintf(stdout, "projdir: %v\n", dir) - return dir, nil + pioutil.MustFprintf(stdout, "projdir: %v\n", targetDir) + return targetDir, nil +} + +func writeCommandOutput(commandName, runDir string, output []byte) (string, error) { + logFileDir := filepath.Join(runDir, "command-output") + if err := os.MkdirAll(logFileDir, 0700); err != nil { + return "", errors.Wrapf(err, "Failed to create '%s'", logFileDir) + } + + logFile := filepath.Join(logFileDir, commandName+uniqueSuffix()+".log") + + if err := ioutil.WriteFile(logFile, output, 0644); err != nil { + return "", errors.Wrapf(err, "Failed to write '%s'", logFile) + } + + return logFile, nil } // RunCommand executes the specified command and additional arguments, wrapping any output in the @@ -680,74 +726,45 @@ func RunCommand(t *testing.T, name string, args []string, wd string, opts *Progr finished = true if runerr != nil { pioutil.MustFprintf(opts.Stderr, "Invoke '%v' failed: %s\n", command, cmdutil.DetailedError(runerr)) - if !opts.Verbose { - pioutil.MustFprintf(opts.Stderr, "%s\n", string(runout)) + } + + // If we collected any program output, write it to a log file -- success or failure. + if len(runout) > 0 { + if logFile, err := writeCommandOutput(name, wd, runout); err != nil { + pioutil.MustFprintf(opts.Stderr, "Failed to write output: %v\n", err) + } else { + pioutil.MustFprintf(opts.Stderr, "Wrote output to %s\n", logFile) } } return runerr } -// prepareProject copies the source directory, src (excluding .pulumi), to a new temporary directory. It then copies -// .pulumi/ and Pulumi.yaml from origin, if any, for edits. The function returns the newly resulting directory. -func prepareProject(t *testing.T, opts *ProgramTestOptions, src, origin string, additive bool) (string, error) { - stackName := opts.GetStackName() - - var dir string - - // If additive, keep the existing directory. Otherwise, create a new one. - if additive { - dir = origin - } else { - // Create a new temp directory. - var err error - dir, err = ioutil.TempDir("", string(stackName)+"-") - if err != nil { - return "", err - } - } - - // Now copy the source into it, ignoring .pulumi/ and Pulumi.yaml if there's an origin. - wdir := workspace.BookkeepingDir - proj := workspace.ProjectFile + ".yaml" - excl := make(map[string]bool) - if origin != "" { - excl[wdir] = true - excl[proj] = true - } - if copyerr := fsutil.CopyFile(dir, src, excl); copyerr != nil { - return "", copyerr - } - - projfile := filepath.Join(dir, proj) - if !additive { - // Now, copy back the original project's .pulumi/ and Pulumi.yaml atop the target. - if origin != "" { - if copyerr := fsutil.CopyFile(projfile, filepath.Join(origin, proj), nil); copyerr != nil { - return "", copyerr - } - if copyerr := fsutil.CopyFile(filepath.Join(dir, wdir), filepath.Join(origin, wdir), nil); copyerr != nil { - return "", copyerr - } - } +// prepareProject adds files in sourceDir to targetDir, then runs setup necessary to get the +// project ready for `pulumi` commands. +func prepareProject(t *testing.T, opts *ProgramTestOptions, sourceDir, targetDir string) error { + // Copy new files to targetDir + if copyerr := fsutil.CopyFile(targetDir, sourceDir, nil); copyerr != nil { + return copyerr } // Write a .yarnrc file to pass --mutex network to all yarn invocations, since tests // may run concurrently and yarn may fail if invoked concurrently // https://github.com/yarnpkg/yarn/issues/683 - yarnrcerr := ioutil.WriteFile(filepath.Join(dir, ".yarnrc"), []byte("--mutex network\n"), 0644) + yarnrcerr := ioutil.WriteFile(filepath.Join(targetDir, ".yarnrc"), []byte("--mutex network\n"), 0644) if yarnrcerr != nil { - return "", yarnrcerr + return yarnrcerr } // Load up the package so we can run Yarn in the correct location. + projfile := filepath.Join(targetDir, workspace.ProjectFile+".yaml") pkginfo, err := engine.ReadPackage(projfile) if err != nil { - return "", err + return err } cwd, _, err := pkginfo.GetPwdMain() if err != nil { - return "", err + return err } if opts.RelativeWorkDir != "" { @@ -758,24 +775,20 @@ func prepareProject(t *testing.T, opts *ProgramTestOptions, src, origin string, if insterr := RunCommand(t, "yarn-install", withOptionalYarnFlags([]string{opts.YarnBin, "install", "--verbose"}), cwd, opts); insterr != nil { - return "", insterr + return insterr } for _, dependency := range opts.Dependencies { if linkerr := RunCommand(t, "yarn-link", withOptionalYarnFlags([]string{opts.YarnBin, "link", dependency}), cwd, opts); linkerr != nil { - return "", linkerr + return linkerr } } // And finally compile it using whatever build steps are in the package.json file. - if builderr := RunCommand(t, + return RunCommand(t, "yarn-build", - withOptionalYarnFlags([]string{opts.YarnBin, "run", "build"}), cwd, opts); builderr != nil { - return "", builderr - } - - return dir, nil + withOptionalYarnFlags([]string{opts.YarnBin, "run", "build"}), cwd, opts) } func withOptionalYarnFlags(args []string) []string { diff --git a/pkg/testing/integration/program_test.go b/pkg/testing/integration/program_test.go index 0cfe5826a..b6bda2cc6 100644 --- a/pkg/testing/integration/program_test.go +++ b/pkg/testing/integration/program_test.go @@ -4,6 +4,10 @@ package integration import ( "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -16,6 +20,37 @@ func TestPrefixer(t *testing.T) { buf := bytes.NewBuffer(byts) prefixer := newPrefixer(buf, "OK: ") _, err := prefixer.Write([]byte("\nsadsada\n\nasdsadsa\nasdsadsa\n")) - contract.Assert(err == nil) + contract.AssertNoError(err) assert.Equal(t, []byte("OK: \nOK: sadsada\nOK: \nOK: asdsadsa\nOK: asdsadsa\n"), buf.Bytes()) } + +// Test that RunCommand writes the command's output to a log file. +func TestRunCommandLog(t *testing.T) { + // Try to find node on the path. We need a program to run, and node is probably + // available on all platforms where we're testing. If it's not found, skip the test. + node, err := exec.LookPath("node") + if err != nil { + t.Skip("Couldn't find Node on PATH") + } + + opts := &ProgramTestOptions{ + Stdout: os.Stdout, + Stderr: os.Stderr, + } + + tempdir, err := ioutil.TempDir("", "test") + contract.AssertNoError(err) + defer os.RemoveAll(tempdir) + + args := []string{node, "-e", "console.log('output from node');"} + err = RunCommand(nil, "node", args, tempdir, opts) + assert.Nil(t, err) + + matches, err := filepath.Glob(filepath.Join(tempdir, "command-output", "node.*")) + assert.Nil(t, err) + assert.Equal(t, 1, len(matches)) + + output, err := ioutil.ReadFile(matches[0]) + assert.Nil(t, err) + assert.Equal(t, "output from node\n", string(output)) +} diff --git a/pkg/tools/lumidl/paths.go b/pkg/tools/lumidl/paths.go index 710e8938e..cd0ff99f9 100644 --- a/pkg/tools/lumidl/paths.go +++ b/pkg/tools/lumidl/paths.go @@ -15,6 +15,6 @@ func RelFilename(root string, prog *loader.Program, p goPos) string { pos := p.Pos() source := prog.Fset.Position(pos).Filename // the source filename.` rel, err := filepath.Rel(root, source) // make it relative to the root. - contract.Assert(err == nil) + contract.AssertNoError(err) return rel } diff --git a/pkg/util/cmdutil/log.go b/pkg/util/cmdutil/log.go index 4d564a747..b407f4b07 100644 --- a/pkg/util/cmdutil/log.go +++ b/pkg/util/cmdutil/log.go @@ -25,10 +25,10 @@ func InitLogging(logToStderr bool, verbose int, logFlow bool) { flag.Parse() if logToStderr { err := flag.Lookup("logtostderr").Value.Set("true") - contract.Assert(err == nil) + contract.AssertNoError(err) } if verbose > 0 { err := flag.Lookup("v").Value.Set(strconv.Itoa(verbose)) - contract.Assert(err == nil) + contract.AssertNoError(err) } } diff --git a/tests/integration/diff/diff_test.go b/tests/integration/diff/diff_test.go index 07a378f4c..395791a1e 100644 --- a/tests/integration/diff/diff_test.go +++ b/tests/integration/diff/diff_test.go @@ -43,10 +43,9 @@ func TestDiffs(t *testing.T) { }, EditDirs: []integration.EditDir{ { - Dir: "step2", - Additive: true, - Stdout: &buf, - Verbose: true, + Dir: "step2", + Stdout: &buf, + Verbose: true, ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { checkpoint := stack.Checkpoint assert.NotNil(t, checkpoint.Latest) @@ -84,10 +83,9 @@ func TestDiffs(t *testing.T) { }, }, { - Dir: "step3", - Additive: true, - Stdout: &buf, - Verbose: true, + Dir: "step3", + Stdout: &buf, + Verbose: true, ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { checkpoint := stack.Checkpoint assert.NotNil(t, checkpoint.Latest) @@ -119,10 +117,9 @@ func TestDiffs(t *testing.T) { }, }, { - Dir: "step4", - Additive: true, - Stdout: &buf, - Verbose: true, + Dir: "step4", + Stdout: &buf, + Verbose: true, ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { checkpoint := stack.Checkpoint assert.NotNil(t, checkpoint.Latest) @@ -151,10 +148,9 @@ func TestDiffs(t *testing.T) { }, }, { - Dir: "step5", - Additive: true, - Stdout: &buf, - Verbose: true, + Dir: "step5", + Stdout: &buf, + Verbose: true, ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { checkpoint := stack.Checkpoint assert.NotNil(t, checkpoint.Latest) diff --git a/tests/integration/protect_resources/protect_test.go b/tests/integration/protect_resources/protect_test.go index d36713a92..6e40aefe6 100644 --- a/tests/integration/protect_resources/protect_test.go +++ b/tests/integration/protect_resources/protect_test.go @@ -29,8 +29,7 @@ func TestSteps(t *testing.T) { }, EditDirs: []integration.EditDir{ { - Dir: "step2", - Additive: true, + Dir: "step2", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { // An update to "eternal"; should still be there. assert.NotNil(t, stackInfo.Checkpoint.Latest) @@ -43,14 +42,12 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step3", - Additive: true, + Dir: "step3", // This step will fail because the resource is protected. ExpectFailure: true, }, { - Dir: "step4", - Additive: true, + Dir: "step4", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { // "eternal" should now be unprotected. assert.NotNil(t, stackInfo.Checkpoint.Latest) @@ -63,8 +60,7 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step5", - Additive: true, + Dir: "step5", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { // Finally, "eternal" should be deleted. assert.NotNil(t, stackInfo.Checkpoint.Latest) diff --git a/tests/integration/steps/steps_test.go b/tests/integration/steps/steps_test.go index f83ea43c0..2fdeef517 100644 --- a/tests/integration/steps/steps_test.go +++ b/tests/integration/steps/steps_test.go @@ -33,8 +33,7 @@ func TestSteps(t *testing.T) { }, EditDirs: []integration.EditDir{ { - Dir: "step2", - Additive: true, + Dir: "step2", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Checkpoint.Latest) assert.Equal(t, 5, len(stackInfo.Checkpoint.Latest.Resources)) @@ -51,8 +50,7 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step3", - Additive: true, + Dir: "step3", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Checkpoint.Latest) assert.Equal(t, 4, len(stackInfo.Checkpoint.Latest.Resources)) @@ -67,8 +65,7 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step4", - Additive: true, + Dir: "step4", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Checkpoint.Latest) assert.Equal(t, 4, len(stackInfo.Checkpoint.Latest.Resources)) @@ -83,8 +80,7 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step5", - Additive: true, + Dir: "step5", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Checkpoint.Latest) // assert.Equal(t, 5, len(checkpoint.Latest.Resources)) @@ -103,8 +99,7 @@ func TestSteps(t *testing.T) { }, }, { - Dir: "step6", - Additive: true, + Dir: "step6", ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { assert.NotNil(t, stackInfo.Checkpoint.Latest) assert.Equal(t, 1, len(stackInfo.Checkpoint.Latest.Resources))