Allow control of uploaded archive root in Pulumi.yaml

Previously, when uploading a projectm to the service, we would only
upload the folder rooted by the Pulumi.yaml for that project. This
worked well, but it meant that customers needed to structure their
code in a way such that Pulumi.yaml was always as the root of their
project, and if they wanted to share common files between two projects
there was no good solution for doing this.

This change introduces an optional piece of metadata, named context,
that can be added to Pulumi.yaml, which allows controlling the root
folder used for computing the root folder to archive from.  When it is
set, it is combined with the location of the Pulumi.yaml file for the
project we are uploading and that folder is uses as the root of what
we upload to the service.

Fixes: #574

pkg/backend/cloud/backend.go

@@ -242,7 +242,12 @@ func (b *cloudBackend) updateStack(action updateKind, stackName tokens.QName, pk
 	if err != nil {
 		return err
 	}
-	updateRequest, err := b.makeProgramUpdateRequest(stackName, pkg, m, opts)
+	context, main, err := getContextAndMain(pkg, root)
+	if err != nil {
+		return err
+	}
+
+	updateRequest, err := b.makeProgramUpdateRequest(stackName, pkg, main, m, opts)
 	if err != nil {
 		return err
 	}
@@ -260,7 +265,7 @@ func (b *cloudBackend) updateStack(action updateKind, stackName tokens.QName, pk
 
 	// Upload the program's contents to the signed URL if appropriate.
 	if action != destroy {
-		err = uploadProgram(pkg, root, updateResponse.UploadURL, true /* print upload size to STDOUT */)
+		err = uploadArchive(context, updateResponse.UploadURL, pkg.UseDefaultIgnores(), true /* show progress */)
 		if err != nil {
 			return err
 		}
@@ -287,17 +292,17 @@ func (b *cloudBackend) updateStack(action updateKind, stackName tokens.QName, pk
 	return nil
 }
 
-// uploadProgram archives the current Pulumi program and uploads it to a signed URL. "current"
+// uploadArchive archives the current Pulumi program and uploads it to a signed URL. "current"
 // meaning whatever Pulumi program is found in the CWD or parent directory.
 // If set, printSize will print the size of the data being uploaded.
-func uploadProgram(pkg *pack.Package, programFolder, uploadURL string, progress bool) error {
+func uploadArchive(context string, uploadURL string, useDefaultIgnores bool, progress bool) error {
 	parsedURL, err := url.Parse(uploadURL)
 	if err != nil {
 		return errors.Wrap(err, "parsing URL")
 	}
 
 	// programPath is the path to the Pulumi.yaml file. Need its parent folder.
-	archiveContents, err := archive.Process(programFolder, pkg.UseDefaultIgnores())
+	archiveContents, err := archive.Process(context, useDefaultIgnores)
 	if err != nil {
 		return errors.Wrap(err, "creating archive")
 	}
@@ -509,8 +514,8 @@ func getCloudProjectIdentifier() (*cloudProjectIdentifier, error) {
 }
 
 // makeProgramUpdateRequest constructs the apitype.UpdateProgramRequest based on the local machine state.
-func (b *cloudBackend) makeProgramUpdateRequest(stackName tokens.QName,
-	pkg *pack.Package, m backend.UpdateMetadata, opts engine.UpdateOptions) (apitype.UpdateProgramRequest, error) {
+func (b *cloudBackend) makeProgramUpdateRequest(stackName tokens.QName, pkg *pack.Package, main string,
+	m backend.UpdateMetadata, opts engine.UpdateOptions) (apitype.UpdateProgramRequest, error) {
 
 	// Convert the configuration into its wire form.
 	cfg, err := state.Configuration(b.d, stackName)
@@ -535,7 +540,7 @@ func (b *cloudBackend) makeProgramUpdateRequest(stackName tokens.QName,
 	return apitype.UpdateProgramRequest{
 		Name:        string(pkg.Name),
 		Runtime:     pkg.Runtime,
-		Main:        pkg.Main,
+		Main:        main,
 		Description: description,
 		Config:      wireConfig,
 		Options: apitype.UpdateOptions{

pkg/backend/cloud/context.go

@@ -0,0 +1,70 @@
+// Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+
+package cloud
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/pulumi/pulumi/pkg/pack"
+	"github.com/pulumi/pulumi/pkg/util/fsutil"
+)
+
+// getContextAndMain computes the root path of the archive as well as the relative path (from the archive root)
+// to the main function. In the case where there is no custom archive root, things are simple, the archive root
+// is the root of the project, and main can remain unchanged. When an context is set, however, we need to do some
+// work:
+//
+// 1. We need to ensure the archive root is "above" the project root.
+// 2. We need to change "main" which was relative to the project root to be relative to the archive root.
+//
+// Note that the relative paths in Pulumi.yaml for Context and Main are always unix style paths, but the returned
+// context is an absolute path, using file system specific seperators. We continue to use a unix style partial path
+// for Main,
+func getContextAndMain(pkg *pack.Package, projectRoot string) (string, string, error) {
+	context, err := filepath.Abs(projectRoot)
+	if err != nil {
+		return "", "", err
+	}
+
+	main := pkg.Main
+
+	if pkg.Context != "" {
+		context, err = filepath.Abs(filepath.Join(context,
+			strings.Replace(pkg.Context, "/", string(filepath.Separator), -1)))
+		if err != nil {
+			return "", "", err
+		}
+
+		if !strings.HasPrefix(projectRoot, context) {
+			return "", "", errors.Errorf("Context directory '%v' is not a parent of '%v'", context, projectRoot)
+		}
+
+		// Walk up to the archive root, starting from the project root, recording the directories we see,
+		// we'll combine these with the existing main value to get a main relative to the root of the archive
+		// which is what the pulumi-service expects. We use fsutil.WalkUp here, so we have to provide a dummy
+		// function which ignores every file we visit.
+		ignoreFileVisitFunc := func(string) bool {
+			// return false so fsutil.Walk does not stop early
+			return false
+		}
+
+		prefix := ""
+		_, err := fsutil.WalkUp(projectRoot, ignoreFileVisitFunc, func(p string) bool {
+			if p != context {
+				prefix = filepath.Base(p) + "/" + prefix
+				return true
+			}
+
+			return false
+		})
+		if err != nil {
+			return "", "", err
+		}
+
+		main = prefix + main
+	}
+
+	return context, main, nil
+}

pkg/backend/cloud/context_test.go

@@ -0,0 +1,98 @@
+// Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+
+package cloud
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pulumi/pulumi/pkg/pack"
+	"github.com/pulumi/pulumi/pkg/util/contract"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNoRootNoMain(t *testing.T) {
+	dir, _ := ioutil.TempDir("", "archive-test")
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+	}()
+
+	context, main, err := getContextAndMain(&pack.Package{}, dir)
+	assert.NoError(t, err)
+	assert.Equal(t, dir, context)
+	assert.Equal(t, "", main)
+}
+
+func TestNoRootMain(t *testing.T) {
+	dir, _ := ioutil.TempDir("", "archive-test")
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+	}()
+
+	testPkg := pack.Package{Main: "foo/bar/baz/"}
+
+	context, main, err := getContextAndMain(&testPkg, dir)
+	assert.NoError(t, err)
+	assert.Equal(t, dir, context)
+	assert.Equal(t, testPkg.Main, main)
+}
+
+func TestRootNoMain(t *testing.T) {
+	dir, _ := ioutil.TempDir("", "archive-test")
+	sub := filepath.Join(dir, "sub1", "sub2", "sub3")
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+	}()
+
+	err := os.MkdirAll(sub, 0700)
+	assert.NoError(t, err, "error creating test directory")
+
+	testPkg := pack.Package{
+		Context: "../../../",
+	}
+
+	context, main, err := getContextAndMain(&testPkg, sub)
+	assert.NoError(t, err)
+	assert.Equal(t, dir, context)
+	assert.Equal(t, "sub1/sub2/sub3/", main)
+}
+
+func TestRootMain(t *testing.T) {
+	dir, _ := ioutil.TempDir("", "archive-test")
+	sub := filepath.Join(dir, "sub1", "sub2", "sub3", "sub4")
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+	}()
+
+	err := os.MkdirAll(sub, 0700)
+	assert.NoError(t, err, "error creating test directory")
+
+	testPkg := pack.Package{
+		Context: "../../../",
+		Main:    "sub4/",
+	}
+
+	context, main, err := getContextAndMain(&testPkg, filepath.Dir(sub))
+	assert.NoError(t, err)
+	assert.Equal(t, dir, context)
+	assert.Equal(t, "sub1/sub2/sub3/sub4/", main)
+}
+
+func TestBadContext(t *testing.T) {
+	dir, _ := ioutil.TempDir("", "archive-test")
+	bad, _ := ioutil.TempDir("", "archive-test")
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+		contract.IgnoreError(os.RemoveAll(bad))
+	}()
+
+	testPkg := pack.Package{
+		Context: bad,
+	}
+
+	_, _, err := getContextAndMain(&testPkg, dir)
+
+	assert.Error(t, err)
+}

pkg/pack/package.go

@@ -37,6 +37,7 @@ type Package struct {
 	Analyzers *Analyzers `json:"analyzers,omitempty" yaml:"analyzers,omitempty"` // any analyzers enabled for this project.
 
 	EncryptionSalt   string `json:"encryptionsalt,omitempty" yaml:"encryptionsalt,omitempty"`     // base64 encoded encryption salt.
+	Context          string `json:"context,omitempty" yaml:"context,omitempty"`                   // an optional path (combined with the on disk location of Pulumi.yaml) to control the data uploaded to the service.
 	NoDefaultIgnores *bool  `json:"nodefaultignores,omitempty" yaml:"nodefaultignores,omitempty"` // true if we should only respect .pulumiignore when archiving
 
 	Config map[tokens.ModuleMember]config.Value `json:"config,omitempty" yaml:"config,omitempty"` // optional config (applies to all stacks).

pkg/util/fsutil/walkup.go

@@ -28,7 +28,6 @@ func WalkUp(path string, walkFn func(string) bool, visitParentFn func(string) bo
 			name := file.Name()
 			path := filepath.Join(curr, name)
 			if walkFn(path) {
-
 				return path, nil
 			}
 		}