Run Dep Ensure when testing Go integration projects

Fixes: #3939

.appveyor.yml

@@ -54,6 +54,8 @@ install:
     set PULUMI_LOCAL_NUGET=C:\Pulumi\nuget
     
     set PULUMI_TEST_OWNER=moolumi
+
+    choco install dep
 - ps: >-
     New-Item -Type Directory "$env:USERPROFILE\go\bin"
 build_script:

.github/workflows/windows-build.yml

@@ -15,6 +15,8 @@ jobs:
         python-version: [3.7]
         dotnet: ['3.1.100']
     runs-on: ${{ matrix.platform }}
+    env:
+      GOPATH: ${{ github.workspace }}
     steps:
     - name: Install DotNet ${{ matrix.dotnet }}
       uses: actions/setup-dotnet@v1
@@ -28,8 +30,6 @@ jobs:
       uses: actions/setup-go@v1
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Install Pulumi CLI
-      uses: pulumi/action-install-pulumi-cli@releases/v1
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
@@ -38,28 +38,39 @@ jobs:
       run: |
         pip3 install pyenv-win
         pip3 install pipenv
+    - name: Install Go Dep
+      shell: bash
+      run: |
+        echo "::add-path::$(go env GOPATH)/bin"
+        go get -v github.com/golang/dep/cmd/dep
     - name: Set Build Env Vars
+      shell: bash
+      run: |
+        echo "::set-env name=PULUMI_TEST_OWNER::moolumi"
+        echo "::set-env name=PULUMI_LOCAL_NUGET::D:\\Pulumi\\nuget"
+        echo "::set-env name=PULUMI_ACCESS_TOKEN::${{ secrets.PULUMI_ACCESS_TOKEN }}"
+        echo "::set-env name=PULUMI_ACCESS_TOKEN::${{ secrets.PULUMI_ACCESS_TOKEN }}"
+        echo "::add-path::D:\\Pulumi\\bin"
+    - name: Set AWS Env Vars
       uses: allenevans/set-env@v1.0.0
       with:
-        PULUMI_TEST_OWNER: "moolumi"
-        PULUMI_LOCAL_NUGET: "D:\\Pulumi\\nuget"
-        PULUMI_ROOT: "D:\\Pulumi"
-        PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
-        DOTNET_CLI_TELEMETRY_OPTOUT: 1
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
       with:
-        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
     - name: Get dependencies
       run: |
+        cd src\github.com\${{ github.repository }}
         set GOMODULE111=on
         go mod tidy
         go mod vendor
     - name: Build Pulumi
       run: |
+        cd src\github.com\${{ github.repository }}
         dotnet msbuild /t:Build /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi"
     - name: Run Pulumi Tests
       run: |
+        cd src\github.com\${{ github.repository }}
         dotnet msbuild /t:Tests /v:Detailed build.proj /p:PulumiRoot="D:\\Pulumi"

build.proj

@@ -284,7 +284,7 @@
       <Output TaskParameter="ExitCode" PropertyName="ExamplesTestExitCode" />
     </Exec>
 
-    <Error Text="examples tests (.\examples) failed" 
+    <Error Text="examples tests (.\examples) failed"
            Condition="$(ExamplesTestExitCode) != 0"/>
 
     <Exec Command="go test -v -timeout 40m -cover -parallel $(TestParallelism) .\tests\integration"

pkg/testing/environment.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/pulumi/pulumi/pkg/tools"
 	"github.com/pulumi/pulumi/pkg/util/fsutil"
 	"github.com/stretchr/testify/assert"
 )
@@ -58,6 +59,21 @@ func WriteYarnRCForTest(root string) error {
 		[]byte("--mutex network\n--network-concurrency 1\n"), 0644)
 }
 
+// NewGoEnvironment returns a new Environment object, located in a GOPATH temp directory.
+func NewGoEnvironment(t *testing.T) *Environment {
+	testRoot, err := tools.CreateTemporaryGoFolder("test-env")
+	if err != nil {
+		t.Errorf("error creating test directory %s", err)
+	}
+
+	t.Logf("Created new go test environment")
+	return &Environment{
+		T:        t,
+		RootPath: testRoot,
+		CWD:      testRoot,
+	}
+}
+
 // NewEnvironment returns a new Environment object, located in a temp directory.
 func NewEnvironment(t *testing.T) *Environment {
 	root, err := ioutil.TempDir("", "test-env")

pkg/testing/integration/program.go

@@ -47,6 +47,7 @@ import (
 	"github.com/pulumi/pulumi/pkg/resource/stack"
 	pulumi_testing "github.com/pulumi/pulumi/pkg/testing"
 	"github.com/pulumi/pulumi/pkg/tokens"
+	"github.com/pulumi/pulumi/pkg/tools"
 	"github.com/pulumi/pulumi/pkg/util/ciutil"
 	"github.com/pulumi/pulumi/pkg/util/contract"
 	"github.com/pulumi/pulumi/pkg/util/fsutil"
@@ -248,6 +249,8 @@ type ProgramTestOptions struct {
 	PipenvBin string
 	// DotNetBin is a location of a `dotnet` executable to be run.  Taken from the $PATH if missing.
 	DotNetBin string
+	// goDepBin is the location of a `dep` executable to be run. Taken from $PATH if missing.
+	GoDepBin string
 
 	// Additional environment variables to pass for each command we run.
 	Env []string
@@ -450,6 +453,9 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt
 	if overrides.PipenvBin != "" {
 		opts.PipenvBin = overrides.PipenvBin
 	}
+	if overrides.GoDepBin != "" {
+		opts.GoDepBin = overrides.GoDepBin
+	}
 	if overrides.Env != nil {
 		opts.Env = append(opts.Env, overrides.Env...)
 	}
@@ -629,6 +635,7 @@ type ProgramTester struct {
 	goBin        string              // the `go` binary we are using.
 	pipenvBin    string              // The `pipenv` binary we are using.
 	dotNetBin    string              // the `dotnet` binary we are using.
+	goDepBin     string              // the `goDep` binary we are using.
 	eventLog     string              // The path to the event log for this test.
 	maxStepTries int                 // The maximum number of times to retry a failed pulumi step.
 	tmpdir       string              // the temporary directory we use for our test environment
@@ -676,6 +683,10 @@ func (pt *ProgramTester) getDotNetBin() (string, error) {
 	return getCmdBin(&pt.dotNetBin, "dotnet", pt.opts.DotNetBin)
 }
 
+func (pt *ProgramTester) getGoDepBin() (string, error) {
+	return getCmdBin(&pt.goDepBin, "dep", pt.opts.GoDepBin)
+}
+
 func (pt *ProgramTester) pulumiCmd(args []string) ([]string, error) {
 	bin, err := pt.getBin()
 	if err != nil {
@@ -894,7 +905,7 @@ func (pt *ProgramTester) TestCleanUp() {
 func (pt *ProgramTester) TestLifeCycleInitAndDestroy() error {
 	err := pt.TestLifeCyclePrepare()
 	if err != nil {
-		return errors.Wrap(err, "copying test to temp dir")
+		return errors.Wrapf(err, "copying test to temp dir %s", pt.tmpdir)
 	}
 
 	pt.TestFinished = false
@@ -1448,27 +1459,30 @@ func (pt *ProgramTester) copyTestToTemporaryDirectory() (string, string, error)
 	}
 	fprintf(pt.opts.Stdout, "pulumi: %v\n", bin)
 
-	// For most projects, we will copy to a temporary directory.  For Go projects, however, we must not perturb
-	// the source layout, due to GOPATH and vendoring.  So, skip it for Go.
+	stackName := string(pt.opts.GetStackName())
+
+	// For most projects, we will copy to a temporary directory.  For Go projects, however, we must create
+	// a folder structure that adheres to GOPATH requirements
 	var tmpdir, projdir string
 	if projinfo.Proj.Runtime.Name() == "go" {
-		projdir = projinfo.Root
+		targetDir, err := tools.CreateTemporaryGoFolder("stackName")
+		if err != nil {
+			return "", "", errors.Wrap(err, "Couldn't create temporary directory")
+		}
+		tmpdir = targetDir
+		projdir = targetDir
 	} else {
-		stackName := string(pt.opts.GetStackName())
 		targetDir, tempErr := ioutil.TempDir("", stackName+"-")
 		if tempErr != nil {
 			return "", "", errors.Wrap(tempErr, "Couldn't create temporary directory")
 		}
-
-		// Copy the source project.
-		if copyErr := fsutil.CopyFile(targetDir, sourceDir, nil); copyErr != nil {
-			return "", "", copyErr
-		}
-
-		// Set tmpdir so that the caller will clean up afterwards.
 		tmpdir = targetDir
 		projdir = targetDir
 	}
+	// Copy the source project.
+	if copyErr := fsutil.CopyFile(tmpdir, sourceDir, nil); copyErr != nil {
+		return "", "", copyErr
+	}
 	projinfo.Root = projdir
 
 	err = pt.prepareProject(projinfo)
@@ -1693,6 +1707,10 @@ func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error {
 	if err != nil {
 		return errors.Wrap(err, "locating `go` binary")
 	}
+	goDepBin, err := pt.getGoDepBin()
+	if err != nil {
+		return errors.Wrap(err, "locating `dep` binary")
+	}
 
 	// Ensure GOPATH is known.
 	gopath := os.Getenv("GOPATH")
@@ -1710,13 +1728,34 @@ func (pt *ProgramTester) prepareGoProject(projinfo *engine.Projinfo) error {
 		return err
 	}
 
-	// skip building if the 'go run' invocation path is requested.
-	if !pt.opts.RunBuild {
-		return nil
+	// We only need to run dep-ensure if there is a Gopkg.toml file
+	_, err = os.Stat(filepath.Join(cwd, "Gopkg.toml"))
+	if err == nil {
+		err = pt.runCommand("dep-ensure", []string{goDepBin, "ensure", "-v"}, cwd)
+		if err != nil {
+			return fmt.Errorf("error installing go dependencies: %w", err)
+		}
 	}
 
+	// In our go tests, there seems to be an issue where we *need* to make a build before we
+	// actually run the tests. If a build isn't made, then we get an issue along the lines of
+	// `cannot import absolute path`. We will investigate this and add back in the respect
+	// for the runBuild option
 	outBin := filepath.Join(gopath, "bin", string(projinfo.Proj.Name))
-	return pt.runCommand("go-build", []string{goBin, "build", "-o", outBin, "."}, cwd)
+	if runtime.GOOS == "windows" {
+		outBin = fmt.Sprintf("%s.exe", outBin)
+	}
+	err = pt.runCommand("go-build", []string{goBin, "build", "-o", outBin, "."}, cwd)
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(outBin)
+	if err != nil {
+		return fmt.Errorf("error finding built application artifact: %w", err)
+	}
+
+	return nil
 }
 
 // prepareDotNetProject runs setup necessary to get a .NET project ready for `pulumi` commands.

pkg/tools/paths.go

@@ -15,8 +15,11 @@
 package tools
 
 import (
+	"fmt"
 	"os"
+	"os/user"
 	"path/filepath"
+	"time"
 )
 
 // EnsureDir ensures that a target directory exists (like `mkdir -p`), returning a non-nil error if any problem occurs.
@@ -28,3 +31,23 @@ func EnsureDir(dir string) error {
 func EnsureFileDir(path string) error {
 	return EnsureDir(filepath.Dir(path))
 }
+
+func CreateTemporaryGoFolder(prefix string) (string, error) {
+	gopath := os.Getenv("GOPATH")
+	if gopath == "" {
+		usr, userErr := user.Current()
+		if userErr != nil {
+			return "", userErr
+		}
+		gopath = filepath.Join(usr.HomeDir, "go")
+	}
+
+	folder := fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
+	testRoot := filepath.Join(gopath, "src", folder)
+	err := EnsureDir(testRoot)
+	if err != nil {
+		return "", err
+	}
+
+	return testRoot, err
+}

sdk/go/pulumi-language-go/main.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"syscall"
 
 	pbempty "github.com/golang/protobuf/ptypes/empty"
@@ -97,6 +98,9 @@ const unableToFindProgramTemplate = "unable to find program: %s"
 // findProgram attempts to find the needed program in various locations on the
 // filesystem, eventually resorting to searching in $PATH.
 func findProgram(program string) (string, error) {
+	if runtime.GOOS == "windows" {
+		program = fmt.Sprintf("%s.exe", program)
+	}
 
 	// look in the same directory
 	cwd, err := os.Getwd()

tests/integration/empty/gorun_main/Pulumi.yaml

@@ -1,4 +1,4 @@
 name: emptygorunmain
 description: An empty Go Pulumi program.
 runtime: go
-main: ../gorun_main_src/
+main: ./gorun_main_src/

tests/integration/stack_reference/go/main.go

@@ -5,8 +5,6 @@ package main
 import (
 	"fmt"
 
-	"github.com/pkg/errors"
-
 	"github.com/pulumi/pulumi/sdk/go/pulumi"
 	"github.com/pulumi/pulumi/sdk/go/pulumi/config"
 )
@@ -20,7 +18,7 @@ func main() {
 		_, err := pulumi.NewStackReference(ctx, slug, nil)
 
 		if err != nil {
-			return errors.Wrap(err, "Error reading stack reference.")
+			return fmt.Errorf("error reading stack reference: %v", err)
 		}
 		ctx.Export("val",
 			pulumi.StringArray([]pulumi.StringInput{pulumi.String("a"), pulumi.String("b")}))

tests/integration/stack_reference/go/step1/main.go

@@ -5,8 +5,6 @@ package main
 import (
 	"fmt"
 
-	"github.com/pkg/errors"
-
 	"github.com/pulumi/pulumi/sdk/go/pulumi"
 	"github.com/pulumi/pulumi/sdk/go/pulumi/config"
 )
@@ -21,7 +19,7 @@ func main() {
 		stackRef, err := pulumi.NewStackReference(ctx, slug, nil)
 
 		if err != nil {
-			return errors.Wrap(err, "Error reading stack reference.")
+			return fmt.Errorf("error reading stack reference: %v", err)
 		}
 
 		val := pulumi.StringArrayOutput(stackRef.GetOutput(pulumi.String("val")))
@@ -31,8 +29,8 @@ func main() {
 
 		_ = val.ApplyStringArray(func(v []string) ([]string, error) {
 			if len(v) != 2 || v[0] != "a" || v[1] != "b" {
-				errChan <- errors.Errorf("Invalid result")
-				return nil, errors.Errorf("Invalid result")
+				errChan <- fmt.Errorf("invalid result")
+				return nil, fmt.Errorf("invalid result")
 			}
 			results <- v
 			return v, nil

tests/integration/stack_reference/go/step2/main.go

@@ -5,8 +5,6 @@ package main
 import (
 	"fmt"
 
-	"github.com/pkg/errors"
-
 	"github.com/pulumi/pulumi/sdk/go/pulumi"
 	"github.com/pulumi/pulumi/sdk/go/pulumi/config"
 )
@@ -22,7 +20,7 @@ func main() {
 		stackRef, err := pulumi.NewStackReference(ctx, slug, nil)
 
 		if err != nil {
-			return errors.Wrap(err, "Error reading stack reference.")
+			return fmt.Errorf("error reading stack reference: %v", err)
 		}
 
 		val := pulumi.StringArrayOutput(stackRef.GetOutput(pulumi.String("val2")))
@@ -34,8 +32,8 @@ func main() {
 		_ = val.ApplyStringArray(func(v []string) ([]string, error) {
 
 			if len(v) != 2 || v[0] != "a" || v[1] != "b" {
-				errChan <- errors.Errorf("Invalid result")
-				return nil, errors.Errorf("Invalid result")
+				errChan <- fmt.Errorf("invalid result")
+				return nil, fmt.Errorf("invalid result")
 			}
 			results <- v
 			return v, nil
@@ -44,7 +42,7 @@ func main() {
 			select {
 			case s := <-secret:
 				if !s {
-					return errors.Errorf("Error, stack export should be marked as secret!!!")
+					return fmt.Errorf("error, stack export should be marked as secret")
 				}
 				break
 			case err = <-errChan: