Support .pulumiignore

When deploying a project via the Pulumi.com service, we have to upload
the entire "context" of your project to Pulumi.com. The context of the
program is all files in the directory tree rooted by the `Pulumi.yaml`
file, which will often contain stuff we don't want to upload, but
previously we had no control over what would be updated (and so folks
would do hacky things like delete folders before running `pulumi
update`).

This change adds support for `.pulumiignore` files which should behave
like `.gitignore`. In addition, we were not previously compressing
files when we added them to the zip archive we uploaded and now.

By default, every .pulumiignore file is treated as if it had an
exclusion for `.git/` at the top of the file (users can override this
by adding an explicit `!.git/` to their file) since it is very
unlikely for there to ever be a reason to upload the .git folder to
the service.

Fixes pulumi/pulumi-service#122

Gopkg.lock

@@ -108,6 +108,12 @@
   packages = ["."]
   revision = "2ab6b7470a54bfa9b5b0289f9b4e8fc4839838f7"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/sabhiram/go-gitignore"
+  packages = ["."]
+  revision = "362f9845770f1606d61ba3ddf9cfb1f0780d2ffe"
+
 [[projects]]
   branch = "master"
   name = "github.com/sergi/go-diff"
@@ -213,6 +219,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "86c24eff162018f59f600e5474683bbaca9e4267082184a9d8ee5f4ef45cbc89"
+  inputs-digest = "bec14dd18d0cbcf3d8bb88226e75fa1aa455e0e2a2175655722c120046949ced"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -52,3 +52,8 @@
 [[constraint]]
   branch = "master"
   name = "github.com/sergi/go-diff"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/sabhiram/go-gitignore"
+

pkg/util/archive/archive.go

@@ -8,11 +8,17 @@ package archive
 import (
 	"archive/zip"
 	"bytes"
-	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"strings"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/pulumi/pulumi/pkg/util/contract"
+	"github.com/pulumi/pulumi/pkg/workspace"
+	"github.com/sabhiram/go-gitignore"
 )
 
 // Process returns an in-memory buffer with the archived contents of the provided file path.
@@ -20,68 +26,140 @@ func Process(path string) (*bytes.Buffer, error) {
 	buffer := &bytes.Buffer{}
 	writer := zip.NewWriter(buffer)
 
-	if err := addPathToZip(writer, path, path); err != nil {
+	// We trim `path` from the pathname of every file we add to the zip, but we actaually
+	// want to ensure the files directly under `path` are not added with a path prefix,
+	// so we add an extra os.PathSeparator here to the end of the string if it doesn't
+	// already end with one.
+	if !os.IsPathSeparator(path[len(path)-1]) {
+		path = path + string(os.PathSeparator)
+	}
+
+	if err := addDirectoryToZip(writer, path, path, nil); err != nil {
 		return nil, err
 	}
 	if err := writer.Close(); err != nil {
 		return nil, err
 	}
 
+	glog.V(5).Infof("project archive is %v bytes", buffer.Len())
+
 	return buffer, nil
 }
 
-// addPathToZip adds all the files in a given directory to a zip archive. All files in the archive are relative to the
-// root path. As a result, path must be underneath root.
-func addPathToZip(writer *zip.Writer, root, p string) error {
-	if !strings.HasPrefix(p, root) {
-		return fmt.Errorf("'%s' is not underneath '%s'", p, root)
-	}
+func addDirectoryToZip(writer *zip.Writer, root string, dir string, ignores *ignoreState) error {
+	ignoreFilePath := path.Join(dir, workspace.IgnoreFile)
 
-	file, err := os.Open(p)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = file.Close()
-	}()
+	// If there is an ignorefile, process it before looking at any child paths.
+	if stat, err := os.Stat(ignoreFilePath); err == nil && !stat.IsDir() {
+		glog.V(9).Infof("processing ignore file in %v", dir)
 
-	stat, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
-	h, err := zip.FileInfoHeader(stat)
-	if err != nil {
-		return err
-	}
-	// Strip out the root prefix from the file we put into the archive.
-	h.Name = strings.TrimPrefix(p, root)
-
-	if stat.IsDir() {
-		h.Name += "/"
-	}
-
-	w, err := writer.CreateHeader(h)
-	if err != nil {
-		return err
-	}
-
-	if !stat.IsDir() {
-		if _, err = io.Copy(w, file); err != nil {
-			return err
-		}
-	} else {
-		names, err := file.Readdirnames(-1)
+		ignore, err := readIgnoreFile(ignoreFilePath)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "could not read ignore file in %v", dir)
 		}
 
-		for _, n := range names {
-			if err := addPathToZip(writer, root, path.Join(p, n)); err != nil {
+		ignores = ignores.Append(dir, *ignore)
+	}
+
+	file, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	// No defer because we want to close file as soon as possible (right after we call Readdir).
+
+	infos, err := file.Readdir(-1)
+	contract.IgnoreClose(file)
+	if err != nil {
+		return err
+	}
+
+	for _, info := range infos {
+		fullName := path.Join(dir, info.Name())
+
+		if !info.IsDir() && ignores.IsIgnored(fullName) {
+			glog.V(9).Infof("skip archiving of %v due to ignore file", fullName)
+			continue
+		}
+
+		// Resolve symlinks (Readdir above calls os.Lstat which does not follow symlinks).
+		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+			info, err = os.Stat(fullName)
+			if err != nil {
 				return err
 			}
 		}
+
+		if info.Mode().IsDir() {
+			err := addDirectoryToZip(writer, root, fullName, ignores)
+			if err != nil {
+				return err
+			}
+		} else if info.Mode().IsRegular() {
+			glog.V(9).Infof("adding %v to archive", fullName)
+
+			w, err := writer.Create(convertPathsForZip(strings.TrimPrefix(fullName, root)))
+			if err != nil {
+				return err
+			}
+
+			file, err := os.Open(fullName)
+			if err != nil {
+				return err
+			}
+			// no defer because we want to close file as soon as possible (right after we call Copy)
+
+			_, err = io.Copy(w, file)
+			contract.IgnoreClose(file)
+			if err != nil {
+				return err
+			}
+		} else {
+			glog.V(9).Infof("ignoring special file %v with mode %v", fullName, info.Mode())
+		}
 	}
 
 	return nil
 }
+
+// convertPathsForZip ensures that '/' is uses at the path separator in zip files.
+func convertPathsForZip(path string) string {
+	if os.PathSeparator != '/' {
+		return strings.Replace(path, string(os.PathSeparator), "/", -1)
+	}
+
+	return path
+}
+
+func readIgnoreFile(path string) (*ignore.GitIgnore, error) {
+	buf, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	patterns := []string{".git/"}
+	patterns = append(patterns, strings.Split(string(buf), "\n")...)
+
+	return ignore.CompileIgnoreLines(patterns...)
+}
+
+type ignoreState struct {
+	path    string
+	ignorer ignore.GitIgnore
+	next    *ignoreState
+}
+
+func (s *ignoreState) Append(path string, ignorer ignore.GitIgnore) *ignoreState {
+	return &ignoreState{path: path, ignorer: ignorer, next: s}
+}
+
+func (s *ignoreState) IsIgnored(path string) bool {
+	if s == nil {
+		return false
+	}
+
+	if s.ignorer.MatchesPath(strings.TrimPrefix(path, s.path)) {
+		return true
+	}
+
+	return s.next.IsIgnored(path)
+}

pkg/util/archive/archive_test.go

@@ -0,0 +1,108 @@
+// Copyright 2016-2017, Pulumi Corporation.  All rights reserved.
+package archive
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"testing"
+
+	"github.com/pulumi/pulumi/pkg/util/contract"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIngoreSimple(t *testing.T) {
+	doArchiveTest(t,
+		fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true},
+		fileContents{name: "included.txt", shouldRetain: true},
+		fileContents{name: "node_modules/included.txt", shouldRetain: true},
+		fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false},
+		fileContents{name: "node_modules/pulumi/excluded/excluded.txt", shouldRetain: false})
+}
+
+func TestIgnoreNegate(t *testing.T) {
+	doArchiveTest(t,
+		fileContents{name: ".pulumiignore", contents: []byte("/*\n!/foo\n/foo/*\n!/foo/bar"), shouldRetain: false},
+		fileContents{name: "excluded.txt", shouldRetain: false},
+		fileContents{name: "foo/excluded.txt", shouldRetain: false},
+		fileContents{name: "foo/baz/exlcuded.txt", shouldRetain: false},
+		fileContents{name: "foo/bar/included.txt", shouldRetain: true})
+}
+
+func TestNested(t *testing.T) {
+	doArchiveTest(t,
+		fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true},
+		fileContents{name: "node_modules/.pulumiignore", contents: []byte("@pulumi/"), shouldRetain: true},
+		fileContents{name: "included.txt", shouldRetain: true},
+		fileContents{name: "node_modules/included.txt", shouldRetain: true},
+		fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false},
+		fileContents{name: "node_modules/@pulumi/pulumi-cloud/excluded.txt", shouldRetain: false})
+}
+
+func doArchiveTest(t *testing.T, files ...fileContents) {
+	archive, err := archiveContents(files...)
+	assert.NoError(t, err)
+
+	fmt.Println(archive.Len())
+
+	r, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len()))
+	assert.NoError(t, err)
+
+	checkFiles(t, files, r.File)
+}
+
+func archiveContents(files ...fileContents) (*bytes.Buffer, error) {
+	dir, err := ioutil.TempDir("", "archive-test")
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		contract.IgnoreError(os.RemoveAll(dir))
+	}()
+
+	for _, file := range files {
+		err := os.MkdirAll(path.Dir(path.Join(dir, file.name)), 0755)
+		if err != nil {
+			return nil, err
+		}
+
+		err = ioutil.WriteFile(path.Join(dir, file.name), file.contents, 0644)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return Process(dir)
+}
+
+func checkFiles(t *testing.T, expected []fileContents, actual []*zip.File) {
+	var expectedFiles []string
+	var actualFiles []string
+
+	for _, f := range expected {
+		if f.shouldRetain {
+			expectedFiles = append(expectedFiles, f.name)
+		}
+	}
+
+	for _, f := range actual {
+		actualFiles = append(actualFiles, f.Name)
+	}
+
+	sort.Strings(expectedFiles)
+	sort.Strings(actualFiles)
+
+	assert.Equal(t, expectedFiles, actualFiles)
+}
+
+type fileContents struct {
+	name         string
+	contents     []byte
+	shouldRetain bool
+}

pkg/workspace/paths.go

@@ -20,6 +20,7 @@ const WorkspaceDir = "workspaces"      // the name of the directory that holds w
 const RepoFile = "settings.json"       // the name of the file that holds information specific to the entire repository.
 const ConfigDir = "config"             // the name of the folder that holds local configuration information.
 const WorkspaceFile = "workspace.json" // the name of the file that holds workspace information.
+const IgnoreFile = ".pulumiignore"     // the name of the file that we use to control what information us uploaded to the service
 
 // DetectPackage locates the closest package from the given path, searching "upwards" in the directory hierarchy.  If no
 // Project is found, an empty path is returned.  If problems are detected, they are logged to the diag.Sink.