maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' of github.com:pulumi/pulumi into resource_parenting_lite

Browse source
This commit is contained in:
joeduffy 2017-11-29 12:14:39 -08:00
parent a4c7c05e27 ec5e39fb73
commit 1a112535af
10 changed files with 328 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
build/common.mk
View file

@ -156,9 +156,10 @@ install::
mkdir -p "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)"
cp -r bin/. "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)"
cp package.json "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)"
cp yarn.lock "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)"
rm -rf "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)/node_modules"
cd "$(PULUMI_NODE_MODULES)/$(NODE_MODULE_NAME)" && \
yarn install --production && \
yarn install --offline --production && \
(yarn unlink > /dev/null 2>&1 || true) && \
yarn link
endif

14
cmd/backend_cloud.go
View file

@ -182,20 +182,18 @@ func updateStack(kind updateKind, stackName tokens.QName, debug bool) error {
// meaning whatever Pulumi program is found in the CWD or parent directory.
// If set, printSize will print the size of the data being uploaded.
func uploadProgram(uploadURL string, printSize bool) error {
cwd, err := os.Getwd()
programPath, err := getPackageFilePath()
if err != nil {
return errors.Wrap(err, "getting working directory")
return err
}
programPath, err := workspace.DetectPackage(cwd)
pkg, err := getPackage()
if err != nil {
return errors.Wrap(err, "looking for Pulumi package")
}
if programPath == "" {
return errors.New("no Pulumi package found")
return err
}
// programPath is the path to the Pulumi.yaml file. Need its parent folder.
programFolder := filepath.Dir(programPath)
archiveContents, err := archive.Process(programFolder)
archiveContents, err := archive.Process(programFolder, pkg.UseDefaultIgnores())
if err != nil {
return errors.Wrap(err, "creating archive")
}

38
cmd/debug_cmds.go
View file

@ -4,46 +4,62 @@ package cmd
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/util/archive"
"github.com/pulumi/pulumi/pkg/util/cmdutil"
"github.com/pulumi/pulumi/pkg/workspace"
"github.com/spf13/cobra"
)
// newArchiveCommand creates a command which just builds the archive we would ship to Pulumi.com to
// do a deployment.
func newArchiveCommand() *cobra.Command {
var forceNoDefaultIgnores bool
var forceDefaultIgnores bool
cmd := &cobra.Command{
Use: "archive <path-to-archive>",
Short: "create an archive suitable for deployment",
Args: cobra.ExactArgs(1),
Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error {
cwd, err := os.Getwd()
if err != nil {
return errors.Wrap(err, "getting working directory")
if forceDefaultIgnores && forceNoDefaultIgnores {
return errors.New("can't specify --no-default-ignores and --default-ignores at the same time")
}
programPath, err := workspace.DetectPackage(cwd)
programPath, err := getPackageFilePath()
if err != nil {
return errors.Wrap(err, "looking for Pulumi package")
return err
}
if programPath == "" {
return errors.New("no Pulumi package found")
pkg, err := getPackage()
if err != nil {
return err
}
useDeafultIgnores := pkg.UseDefaultIgnores()
if forceDefaultIgnores {
useDeafultIgnores = true
} else if forceNoDefaultIgnores {
useDeafultIgnores = false
}
// programPath is the path to the Pulumi.yaml file. Need its parent folder.
programFolder := filepath.Dir(programPath)
archiveContents, err := archive.Process(programFolder)
archiveContents, err := archive.Process(programFolder, useDeafultIgnores)
if err != nil {
return err
return errors.Wrap(err, "creating archive")
}
return ioutil.WriteFile(args[0], archiveContents.Bytes(), 0644)
}),
}
cmd.PersistentFlags().BoolVar(
&forceNoDefaultIgnores, "--no-default-ignores", false,
"Do not use default ignores, regardless of Pulumi.yaml")
cmd.PersistentFlags().BoolVar(
&forceDefaultIgnores, "--default-ignores", false,
"Use default ignores, regardless of Pulumi.yaml")
return cmd
}

11
pkg/pack/package.go
View file

@ -37,7 +37,8 @@ type Package struct {
Analyzers *Analyzers `json:"analyzers,omitempty" yaml:"analyzers,omitempty"` // any analyzers enabled for this project.
EncryptionSalt string `json:"encryptionsalt,omitempty" yaml:"encryptionsalt,omitempty"` // base64 encoded encryption salt.
EncryptionSalt string `json:"encryptionsalt,omitempty" yaml:"encryptionsalt,omitempty"` // base64 encoded encryption salt.
NoDefaultIgnores *bool `json:"nodefaultignores,omitempty" yaml:"nodefaultignores,omitempty"` // true if we should only respect .pulumiignore when archiving
Config map[tokens.ModuleMember]config.Value `json:"config,omitempty" yaml:"config,omitempty"` // optional config (applies to all stacks).
@ -67,6 +68,14 @@ func (pkg *Package) Validate() error {
return nil
}
func (pkg *Package) UseDefaultIgnores() bool {
if pkg.NoDefaultIgnores == nil {
return true
}
return !(*pkg.NoDefaultIgnores)
}
// Analyzers is a list of analyzers to run on this project.
type Analyzers []tokens.QName

85
pkg/util/archive/archive.go
View file

@ -9,7 +9,6 @@ import (
"archive/zip"
"bytes"
"io"
"io/ioutil"
"os"
"path"
"strings"
@ -18,11 +17,10 @@ import (
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/workspace"
"github.com/sabhiram/go-gitignore"
)
// Process returns an in-memory buffer with the archived contents of the provided file path.
func Process(path string) (*bytes.Buffer, error) {
func Process(path string, useDefaultExcludes bool) (*bytes.Buffer, error) {
buffer := &bytes.Buffer{}
writer := zip.NewWriter(buffer)
@ -34,7 +32,7 @@ func Process(path string) (*bytes.Buffer, error) {
path = path + string(os.PathSeparator)
}
if err := addDirectoryToZip(writer, path, path, nil); err != nil {
if err := addDirectoryToZip(writer, path, path, useDefaultExcludes, nil); err != nil {
return nil, err
}
if err := writer.Close(); err != nil {
@ -46,19 +44,39 @@ func Process(path string) (*bytes.Buffer, error) {
return buffer, nil
}
func addDirectoryToZip(writer *zip.Writer, root string, dir string, ignores *ignoreState) error {
func addDirectoryToZip(writer *zip.Writer, root string, dir string, useDefaultIgnores bool, ignores *ignoreState) error {
ignoreFilePath := path.Join(dir, workspace.IgnoreFile)
// If there is an ignorefile, process it before looking at any child paths.
if stat, err := os.Stat(ignoreFilePath); err == nil && !stat.IsDir() {
glog.V(9).Infof("processing ignore file in %v", dir)
ignore, err := readIgnoreFile(ignoreFilePath)
ignore, err := newPulumiIgnorerIgnorer(ignoreFilePath)
if err != nil {
return errors.Wrapf(err, "could not read ignore file in %v", dir)
}
ignores = ignores.Append(dir, *ignore)
ignores = ignores.Append(ignore)
}
if useDefaultIgnores {
dotGitPath := path.Join(dir, ".git")
if stat, err := os.Stat(dotGitPath); err == nil {
ignores = ignores.Append(newPathIgnorer(dotGitPath, stat.IsDir()))
}
// If there is a package.json file here, let's build a node_modules ignorer from it.
packageJSONFilePath := path.Join(dir, packageJSONFileName)
if stat, err := os.Stat(packageJSONFilePath); err == nil && !stat.IsDir() {
glog.V(9).Infof("building ignore filter from package.json in %v", dir)
ignore, err := newNodeModulesIgnorer(packageJSONFilePath)
if err != nil {
return errors.Wrapf(err, "could not read ignores from package.json file in %v", dir)
}
ignores = ignores.Append(ignore)
}
}
file, err := os.Open(dir)
@ -90,7 +108,24 @@ func addDirectoryToZip(writer *zip.Writer, root string, dir string, ignores *ign
}
if info.Mode().IsDir() {
err := addDirectoryToZip(writer, root, fullName, ignores)
// Work around an issue that will be addressed by pulumi/pulumi-ppc#95, by ensuring
// our zip files contain directory entries instead of just having files with paths.
// When the PPC fix is everywhere, we can delete this code in favor of just calling
// addDirectoryToZip recursively
zh, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
// Add a trailing slash since this is a directory.
zh.Name = convertPathsForZip(strings.TrimPrefix(fullName, root)) + "/"
_, err = writer.CreateHeader(zh)
if err != nil {
return err
}
err = addDirectoryToZip(writer, root, fullName, useDefaultIgnores, ignores)
if err != nil {
return err
}
@ -129,37 +164,3 @@ func convertPathsForZip(path string) string {
return path
}
func readIgnoreFile(path string) (*ignore.GitIgnore, error) {
buf, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
patterns := []string{".git/"}
patterns = append(patterns, strings.Split(string(buf), "\n")...)
return ignore.CompileIgnoreLines(patterns...)
}
type ignoreState struct {
path string
ignorer ignore.GitIgnore
next *ignoreState
}
func (s *ignoreState) Append(path string, ignorer ignore.GitIgnore) *ignoreState {
return &ignoreState{path: path, ignorer: ignorer, next: s}
}
func (s *ignoreState) IsIgnored(path string) bool {
if s == nil {
return false
}
if s.ignorer.MatchesPath(strings.TrimPrefix(path, s.path)) {
return true
}
return s.next.IsIgnored(path)
}

9
pkg/util/archive/archive_test.go
View file

@ -9,6 +9,7 @@ import (
"os"
"path"
"sort"
"strings"
"testing"
"github.com/pulumi/pulumi/pkg/util/contract"
@ -78,7 +79,7 @@ func archiveContents(files ...fileContents) (*bytes.Buffer, error) {
}
}
return Process(dir)
return Process(dir, false)
}
func checkFiles(t *testing.T, expected []fileContents, actual []*zip.File) {
@ -92,6 +93,12 @@ func checkFiles(t *testing.T, expected []fileContents, actual []*zip.File) {
}
for _, f := range actual {
// Ignore any directories (we only care that the files themselves are correct)
if strings.HasSuffix(f.Name, "/") {
continue
}
actualFiles = append(actualFiles, f.Name)
}

28
pkg/util/archive/ignorer.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
package archive
type ignorer interface {
IsIgnored(f string) bool
}
type ignoreState struct {
ignorer ignorer
next *ignoreState
}
func (s *ignoreState) Append(ignorer ignorer) *ignoreState {
return &ignoreState{ignorer: ignorer, next: s}
}
func (s *ignoreState) IsIgnored(path string) bool {
if s == nil {
return false
}
if s.ignorer.IsIgnored(path) {
return true
}
return s.next.IsIgnored(path)
}

140
pkg/util/archive/ignorer_node_modules.go Normal file
View file

@ -0,0 +1,140 @@
// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
package archive
import (
"container/list"
"encoding/json"
"os"
"path"
"strings"
"github.com/pulumi/pulumi/pkg/util/contract"
)
const nodeModulesDirName = "node_modules"
const packageJSONFileName = "package.json"
// newNodeModulesIgnorer constructs a new ignorer that ignores all files under node_modules that do not belong to packages in the
// transitive closure of the dependencies section of a package.json.
func newNodeModulesIgnorer(packageJSONPath string) (ignorer, error) {
depRoots, err := expandDepenencies(packageJSONPath)
if err != nil {
return nil, err
}
return &nodeModulesIgnorer{root: path.Dir(packageJSONPath), allowed: depRoots}, nil
}
func expandDepenencies(packageJSONPath string) ([]string, error) {
deps, err := getPackageDependencies(packageJSONPath)
if err != nil {
return nil, err
}
queue := list.New()
root := path.Dir(packageJSONPath)
for _, dep := range deps {
queue.PushBack(packageContext{name: dep, root: root})
}
var found struct{}
marked := make(map[string]struct{})
// While we still have packages we haven't processed, visit one.
for queue.Len() > 0 {
elm := queue.Front()
queue.Remove(elm)
ctx := elm.Value.(packageContext)
for curDir := ctx.root; strings.HasPrefix(curDir, root); curDir = path.Dir(curDir) {
candidateModuleFolder := path.Join(curDir, nodeModulesDirName, ctx.name)
if _, err := os.Stat(candidateModuleFolder); err == nil {
_, has := marked[candidateModuleFolder]
marked[candidateModuleFolder] = found
// If we hadn't already seen this module, ensure we also visit its dependencies.
if !has {
deps, err := getPackageDependencies(path.Join(candidateModuleFolder, packageJSONFileName))
if err != nil {
return nil, err
}
for _, dep := range deps {
queue.PushBack(packageContext{name: dep, root: candidateModuleFolder})
}
}
// Don't explore up the file system tree, since we found a match.
break
}
}
}
ret := make([]string, 0, len(marked))
for dir := range marked {
ret = append(ret, dir)
}
return ret, nil
}
func getPackageDependencies(packageJSONPath string) ([]string, error) {
f, err := os.Open(packageJSONPath)
if err != nil {
return nil, err
}
defer func() {
contract.IgnoreError(f.Close())
}()
dec := json.NewDecoder(f)
var file packageJSONFile
err = dec.Decode(&file)
if err != nil {
return nil, err
}
packages := make([]string, 0, len(file.Dependencies))
for dep := range file.Dependencies {
packages = append(packages, dep)
}
return packages, nil
}
type nodeModulesIgnorer struct {
root string // the directory for the node project this ingorer was built from
allowed []string // the list of all node_modules folders that the project needs
}
// MatchesPath for a nodeModulesIngorer can ignore any file under node_modules for the project
// it was constructed from as along as long as it is not in a directory for a module we know
// we must retain.
func (n *nodeModulesIgnorer) IsIgnored(f string) bool {
if !strings.HasPrefix(f, path.Join(n.root, nodeModulesDirName)) {
return false
}
for _, prefix := range n.allowed {
if strings.HasPrefix(f, prefix) {
return false
}
}
return true
}
type packageContext struct {
name string // The name of the package from the package.json.
root string // The directory of the package.json this was found in (this is where we have to start our search).
}
type packageJSONFile struct {
Dependencies map[string]string `json:"dependencies"`
}

34
pkg/util/archive/ignorer_path.go Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
package archive
import (
"os"
"strings"
)
// newPathIgnorer creates an ignorer based that ignores either a single file (when dir is false) or
// and entire directory tree (when dir is true).
func newPathIgnorer(path string, isDir bool) ignorer {
if !isDir {
return &fileIgnorer{path: path}
}
return &directoryIgnorer{path: path + string(os.PathSeparator)}
}
type fileIgnorer struct {
path string
}
func (fi *fileIgnorer) IsIgnored(f string) bool {
return f == fi.path
}
type directoryIgnorer struct {
path string
}
func (di *directoryIgnorer) IsIgnored(f string) bool {
return strings.HasPrefix(f, di.path)
}

30
pkg/util/archive/ignorer_pulumiignore.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
package archive
import (
"path"
"strings"
ignore "github.com/sabhiram/go-gitignore"
)
// newPulumiIgnorerIgnorer creates an ignorer based on the contents of a .pulumiignore file, which
// has the same semantics as a .gitignore file
func newPulumiIgnorerIgnorer(pathToPulumiIgnore string) (ignorer, error) {
gitIgnorer, err := ignore.CompileIgnoreFile(pathToPulumiIgnore)
if err != nil {
return nil, err
}
return &pulumiIgnoreIgnorer{root: path.Dir(pathToPulumiIgnore), ignorer: gitIgnorer}, nil
}
type pulumiIgnoreIgnorer struct {
root string
ignorer *ignore.GitIgnore
}
func (g *pulumiIgnoreIgnorer) IsIgnored(f string) bool {
return g.ignorer.MatchesPath(strings.TrimPrefix(f, g.root))
}