maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Use npm pack for policy publish

Browse source 
Fixes #35.
This commit is contained in:
Alex Clemmer 2019-08-01 11:05:57 -07:00
parent 6360cba588
commit 7070304f81
10 changed files with 173 additions and 655 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
cmd/new.go
View file

@ -18,33 +18,30 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"unicode"
"github.com/pulumi/pulumi/pkg/backend/state"
"github.com/pkg/errors"
"github.com/spf13/cobra"
survey "gopkg.in/AlecAivazis/survey.v1"
surveycore "gopkg.in/AlecAivazis/survey.v1/core"
"github.com/pulumi/pulumi/pkg/apitype"
"github.com/pulumi/pulumi/pkg/backend"
"github.com/pulumi/pulumi/pkg/backend/display"
"github.com/pulumi/pulumi/pkg/backend/httpstate"
"github.com/pulumi/pulumi/pkg/backend/state"
"github.com/pulumi/pulumi/pkg/diag/colors"
"github.com/pulumi/pulumi/pkg/npm"
"github.com/pulumi/pulumi/pkg/resource/config"
"github.com/pulumi/pulumi/pkg/tokens"
"github.com/pulumi/pulumi/pkg/workspace"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/diag/colors"
"github.com/pulumi/pulumi/pkg/util/cmdutil"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/logging"
"github.com/spf13/cobra"
survey "gopkg.in/AlecAivazis/survey.v1"
surveycore "gopkg.in/AlecAivazis/survey.v1/core"
"github.com/pulumi/pulumi/pkg/workspace"
)
// Intentionally disabling here for cleaner err declaration/assignment.
@ -466,15 +463,7 @@ func installDependencies() error {
return err
}
// TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here.
var command string
var c *exec.Cmd
if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
command = "npm install"
// We pass `--loglevel=error` to prevent `npm` from printing warnings about missing
// `description`, `repository`, and `license` fields in the package.json file.
c = exec.Command("npm", "install", "--loglevel=error")
} else {
if !strings.EqualFold(proj.Runtime.Name(), "nodejs") {
return nil
}
@ -482,17 +471,11 @@ func installDependencies() error {
fmt.Println()
// Run the command.
c.Stdout = os.Stdout
c.Stderr = os.Stderr
if err := c.Run(); err != nil {
return errors.Wrapf(err, "installing dependencies; rerun '%s' manually to try again, "+
"then run 'pulumi up' to perform an initial deployment", command)
}
// Ensure the "node_modules" directory exists.
if _, err := os.Stat("node_modules"); os.IsNotExist(err) {
return errors.Errorf("installing dependencies; rerun '%s' manually to try again, "+
"then run 'pulumi up' to perform an initial deployment", command)
// TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here.
err = npm.Install("", os.Stdout, os.Stderr)
if err != nil {
return errors.Wrapf(err, "npm install failed; rerun manually to try again, "+
"then run 'pulumi up' to perform an initial deployment")
}
fmt.Println("Finished installing dependencies")

65
pkg/backend/httpstate/policypack.go
View file

@ -1,12 +1,19 @@
package httpstate
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"github.com/pulumi/pulumi/pkg/util/archive"
"github.com/pulumi/pulumi/pkg/npm"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/apitype"
@ -14,7 +21,6 @@ import (
"github.com/pulumi/pulumi/pkg/backend/httpstate/client"
"github.com/pulumi/pulumi/pkg/engine"
"github.com/pulumi/pulumi/pkg/tokens"
"github.com/pulumi/pulumi/pkg/util/archive"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/result"
"github.com/pulumi/pulumi/pkg/workspace"
@ -48,12 +54,12 @@ func (rp *cloudRequiredPolicy) Install(ctx context.Context) (string, error) {
}
// PolicyPack has not been downloaded and installed. Do this now.
policyPackZip, err := rp.client.DownloadPolicyPack(ctx, policy.PackLocation)
policyPackTarball, err := rp.client.DownloadPolicyPack(ctx, policy.PackLocation)
if err != nil {
return "", err
}
return policyPackPath, installRequiredPolicy(policyPackPath, policyPackZip)
return policyPackPath, installRequiredPolicy(policyPackPath, policyPackTarball)
}
func newCloudBackendPolicyPackReference(
@ -127,9 +133,17 @@ func (pack *cloudPolicyPack) Publish(
fmt.Println("Compressing policy pack")
dirArchive, err := archive.Process(op.Root, false)
if runtime := pack.b.currentProject.Runtime.Name(); !strings.EqualFold(runtime, "nodejs") {
return result.Errorf(
"failed to publish policies becuase Pulumi.yaml requests unsupported runtime %s",
runtime)
}
// TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here.
packTarball, err := npm.Pack(op.PlugCtx.Pwd, os.Stderr)
if err != nil {
return result.FromError(err)
return result.FromError(
errors.Wrapf(err, "could not publish policies because of error running npm pack"))
}
//
@ -138,7 +152,7 @@ func (pack *cloudPolicyPack) Publish(
fmt.Println("Uploading policy pack to Pulumi service")
err = pack.cl.PublishPolicyPack(ctx, pack.ref.orgName, analyzerInfo, dirArchive)
err = pack.cl.PublishPolicyPack(ctx, pack.ref.orgName, analyzerInfo, bytes.NewReader(packTarball))
if err != nil {
return result.FromError(err)
}
@ -150,7 +164,9 @@ func (pack *cloudPolicyPack) Apply(ctx context.Context, op backend.ApplyOperatio
return pack.cl.ApplyPolicyPack(ctx, pack.ref.orgName, string(pack.ref.name), op.Version)
}
func installRequiredPolicy(finalDir string, zip []byte) error {
const npmPackageDir = "package"
func installRequiredPolicy(finalDir string, tarball []byte) error {
// If part of the directory tree is missing, ioutil.TempDir will return an error, so make sure
// the path we're going to create the temporary folder in actually exists.
if err := os.MkdirAll(filepath.Dir(finalDir), 0700); err != nil {
@ -162,14 +178,19 @@ func installRequiredPolicy(finalDir string, zip []byte) error {
return errors.Wrapf(err, "creating plugin directory %s", tempDir)
}
// npm unpacks into a directory called `package`.
tempNPMPkgDir := path.Join(tempDir, npmPackageDir)
if err := os.MkdirAll(tempNPMPkgDir, 0700); err != nil {
return errors.Wrap(err, "creating plugin root")
}
// If we early out of this function, try to remove the temp folder we created.
defer func() {
contract.IgnoreError(os.RemoveAll(tempDir))
}()
// Unzip the file. NOTE: It is important that the `Close` calls in this function complete before
// we try to rename the temp directory. Open file handles here cause issues on Windows.
err = archive.Unzip(zip, tempDir)
// Uncompress the policy pack.
err = archive.Untgz(tarball, tempDir)
if err != nil {
return err
}
@ -179,9 +200,31 @@ func installRequiredPolicy(finalDir string, zip []byte) error {
// If two calls to `plugin install` for the same plugin are racing, the second one will be
// unable to rename the directory. That's OK, just ignore the error. The temp directory created
// as part of the install will be cleaned up when we exit by the defer above.
if err := os.Rename(tempDir, finalDir); err != nil && !os.IsExist(err) {
if err := os.Rename(tempNPMPkgDir, finalDir); err != nil && !os.IsExist(err) {
return errors.Wrap(err, "moving plugin")
}
proj, err := workspace.LoadProject(path.Join(finalDir, "Pulumi.yaml"))
if err != nil {
return errors.Wrapf(err, "failed to load policy project at %s", finalDir)
}
// TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here.
if !strings.EqualFold(proj.Runtime.Name(), "nodejs") {
return fmt.Errorf("unsupported policy runtime %s", proj.Runtime.Name())
}
fmt.Println("Installing dependencies...")
fmt.Println()
// TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here.
err = npm.Install(finalDir, nil, nil)
if err != nil {
return err
}
fmt.Println("Finished installing dependencies")
fmt.Println()
return nil
}

66
pkg/npm/npm.go Normal file
View file

@ -0,0 +1,66 @@
package npm
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"strings"
"github.com/pkg/errors"
)
// Pack runs `npm pack` in the given directory, packaging the Node.js app located there into a
// tarball and returning it as `[]byte`. `stdout` is ignored for the command, as it does not
// generate useful data.
func Pack(dir string, stderr io.Writer) ([]byte, error) {
// We pass `--loglevel=error` to prevent `npm` from printing warnings about missing
// `description`, `repository`, and `license` fields in the package.json file.
c := exec.Command("npm", "pack", "--loglevel=error")
c.Dir = dir
// Run the command. Note that `npm pack` doesn't have the ability to rename the resulting
// filename, since it's meant to be uploaded directly to npm, which means that we have to get
// that information by parsing the output of the command.
var stdout bytes.Buffer
c.Stdout = &stdout
c.Stderr = os.Stderr
if err := c.Run(); err != nil {
return nil, err
}
packfile := strings.TrimSpace(stdout.String())
defer os.Remove(packfile)
packTarball, err := ioutil.ReadFile(packfile)
if err != nil {
return nil, fmt.Errorf(
"npm pack completed successfully but the packed .tar.gz file was not generated")
}
return packTarball, nil
}
// Install runs `npm install` in the given directory, installing the dependencies for the Node.js
// app located there.
func Install(dir string, stdout, stderr io.Writer) error {
// We pass `--loglevel=error` to prevent `npm` from printing warnings about missing
// `description`, `repository`, and `license` fields in the package.json file.
c := exec.Command("npm", "install", "--loglevel=error")
c.Dir = dir
// Run the command.
c.Stdout = stdout
c.Stderr = stderr
if err := c.Run(); err != nil {
return err
}
// Ensure the "node_modules" directory exists.
if _, err := os.Stat("node_modules"); os.IsNotExist(err) {
return errors.Errorf("npm install reported success, but node_modeules directory is missing")
}
return nil
}

196
pkg/util/archive/archive.go
View file

@ -18,191 +18,57 @@
package archive
import (
"archive/zip"
"archive/tar"
"bytes"
"compress/gzip"
"io"
"os"
"path"
"path/filepath"
"sort"
"strings"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/logging"
"github.com/pulumi/pulumi/pkg/workspace"
)
// Process returns an in-memory buffer with the archived contents of the provided file path.
func Process(path string, useDefaultExcludes bool) (*bytes.Buffer, error) {
buffer := &bytes.Buffer{}
writer := zip.NewWriter(buffer)
// We trim `path` from the pathname of every file we add to the zip, but we actaually
// want to ensure the files directly under `path` are not added with a path prefix,
// so we add an extra os.PathSeparator here to the end of the string if it doesn't
// already end with one.
if !os.IsPathSeparator(path[len(path)-1]) {
path = path + string(os.PathSeparator)
}
if err := addDirectoryToZip(writer, path, path, useDefaultExcludes, nil); err != nil {
return nil, err
}
if err := writer.Close(); err != nil {
return nil, err
}
logging.V(5).Infof("project archive is %v bytes", buffer.Len())
return buffer, nil
}
// Unzip a zip file into a specific directory.
func Unzip(archive []byte, dir string) error {
r, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
// Untgz uncompresses a .tar.gz/.tgz file into a specific directory.
func Untgz(tarball []byte, dir string) error {
tarReader := bytes.NewReader(tarball)
gzr, err := gzip.NewReader(tarReader)
if err != nil {
return err
return errors.Wrapf(err, "unzipping")
}
sort.Slice(r.File, func(i, j int) bool { return r.File[i].Name < r.File[j].Name })
for _, file := range r.File {
path := filepath.Join(dir, file.Name) // #nosec
if file.FileInfo().IsDir() {
contract.IgnoreError(os.MkdirAll(path, file.Mode()))
continue
} else {
contract.IgnoreError(os.MkdirAll(filepath.Dir(path), 0700))
r := tar.NewReader(gzr)
for {
header, err := r.Next()
if err == io.EOF {
break
} else if err != nil {
return errors.Wrapf(err, "untarring")
}
fileReader, err := file.Open()
if err != nil {
return errors.Wrapf(err, "Failed to open file")
}
defer fileReader.Close()
path := filepath.Join(dir, header.Name)
targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
if err != nil {
return errors.Wrapf(err, "Failed to open target file")
}
defer targetFile.Close()
if _, err := io.Copy(targetFile, fileReader); err != nil {
return err
}
// Eagerly try to close to avoid having too many open file descriptors for big archives. We
// simply ignore double-close errors.
fileReader.Close()
targetFile.Close()
}
return nil
}
func addDirectoryToZip(writer *zip.Writer, root string, dir string,
useDefaultIgnores bool, ignores *ignoreState) error {
ignoreFilePath := path.Join(dir, workspace.IgnoreFile)
// If there is an ignorefile, process it before looking at any child paths.
if stat, err := os.Stat(ignoreFilePath); err == nil && !stat.IsDir() {
logging.V(9).Infof("processing ignore file in %v", dir)
ignore, err := newPulumiIgnorerIgnorer(ignoreFilePath)
if err != nil {
return errors.Wrapf(err, "could not read ignore file in %v", dir)
}
ignores = ignores.Append(ignore)
}
if useDefaultIgnores {
dotGitPath := path.Join(dir, ".git")
if stat, err := os.Stat(dotGitPath); err == nil {
ignores = ignores.Append(newPathIgnorer(dotGitPath, stat.IsDir()))
}
// If there is a package.json file here, let's build a node_modules ignorer from it.
packageJSONFilePath := path.Join(dir, packageJSONFileName)
if stat, err := os.Stat(packageJSONFilePath); err == nil && !stat.IsDir() {
logging.V(9).Infof("building ignore filter from package.json in %v", dir)
ignore, err := newNodeModulesIgnorer(packageJSONFilePath)
if err != nil {
return errors.Wrapf(err, "could not read ignores from package.json file in %v", dir)
switch header.Typeflag {
case tar.TypeDir:
// Create any directories as needed.
if _, err := os.Stat(path); err != nil {
if err = os.MkdirAll(path, 0700); err != nil {
return errors.Wrapf(err, "untarring dir %s", path)
}
}
ignores = ignores.Append(ignore)
}
}
file, err := os.Open(dir)
if err != nil {
return err
}
// No defer because we want to close file as soon as possible (right after we call Readdir).
infos, err := file.Readdir(-1)
contract.IgnoreClose(file)
if err != nil {
return err
}
for _, info := range infos {
fullName := path.Join(dir, info.Name())
if !info.IsDir() && ignores.IsIgnored(fullName) {
logging.V(9).Infof("skip archiving of %v due to ignore file", fullName)
continue
}
// Resolve symlinks (Readdir above calls os.Lstat which does not follow symlinks).
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
info, err = os.Stat(fullName)
case tar.TypeReg:
// Expand files into the target directory.
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return err
return errors.Wrapf(err, "opening file %s for untar", path)
}
}
if info.Mode().IsDir() {
err = addDirectoryToZip(writer, root, fullName, useDefaultIgnores, ignores)
if err != nil {
return err
defer contract.IgnoreClose(dst)
if _, err = io.Copy(dst, r); err != nil {
return errors.Wrapf(err, "untarring file %s", path)
}
} else if info.Mode().IsRegular() {
logging.V(9).Infof("adding %v to archive", fullName)
w, err := writer.Create(convertPathsForZip(strings.TrimPrefix(fullName, root)))
if err != nil {
return err
}
file, err := os.Open(fullName)
if err != nil {
return err
}
// no defer because we want to close file as soon as possible (right after we call Copy)
_, err = io.Copy(w, file)
contract.IgnoreClose(file)
if err != nil {
return err
}
} else {
logging.V(9).Infof("ignoring special file %v with mode %v", fullName, info.Mode())
default:
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
}
}
return nil
}
// convertPathsForZip ensures that '/' is uses at the path separator in zip files.
func convertPathsForZip(path string) string {
if os.PathSeparator != '/' {
return strings.Replace(path, string(os.PathSeparator), "/", -1)
}
return path
}

128
pkg/util/archive/archive_test.go
View file

@ -1,128 +0,0 @@
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"archive/zip"
"bytes"
"fmt"
"io/ioutil"
"os"
"path"
"sort"
"strings"
"testing"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/stretchr/testify/assert"
)
func TestIngoreSimple(t *testing.T) {
doArchiveTest(t,
fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true},
fileContents{name: "included.txt", shouldRetain: true},
fileContents{name: "node_modules/included.txt", shouldRetain: true},
fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false},
fileContents{name: "node_modules/pulumi/excluded/excluded.txt", shouldRetain: false})
}
func TestIgnoreNegate(t *testing.T) {
doArchiveTest(t,
fileContents{name: ".pulumiignore", contents: []byte("/*\n!/foo\n/foo/*\n!/foo/bar"), shouldRetain: false},
fileContents{name: "excluded.txt", shouldRetain: false},
fileContents{name: "foo/excluded.txt", shouldRetain: false},
fileContents{name: "foo/baz/exlcuded.txt", shouldRetain: false},
fileContents{name: "foo/bar/included.txt", shouldRetain: true})
}
func TestNested(t *testing.T) {
doArchiveTest(t,
fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true},
fileContents{name: "node_modules/.pulumiignore", contents: []byte("@pulumi/"), shouldRetain: true},
fileContents{name: "included.txt", shouldRetain: true},
fileContents{name: "node_modules/included.txt", shouldRetain: true},
fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false},
fileContents{name: "node_modules/@pulumi/pulumi-cloud/excluded.txt", shouldRetain: false})
}
func doArchiveTest(t *testing.T, files ...fileContents) {
archive, err := archiveContents(files...)
assert.NoError(t, err)
fmt.Println(archive.Len())
r, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len()))
assert.NoError(t, err)
checkFiles(t, files, r.File)
}
func archiveContents(files ...fileContents) (*bytes.Buffer, error) {
dir, err := ioutil.TempDir("", "archive-test")
if err != nil {
return nil, err
}
defer func() {
contract.IgnoreError(os.RemoveAll(dir))
}()
for _, file := range files {
err := os.MkdirAll(path.Dir(path.Join(dir, file.name)), 0755)
if err != nil {
return nil, err
}
err = ioutil.WriteFile(path.Join(dir, file.name), file.contents, 0644)
if err != nil {
return nil, err
}
}
return Process(dir, false)
}
func checkFiles(t *testing.T, expected []fileContents, actual []*zip.File) {
var expectedFiles []string
var actualFiles []string
for _, f := range expected {
if f.shouldRetain {
expectedFiles = append(expectedFiles, f.name)
}
}
for _, f := range actual {
// Ignore any directories (we only care that the files themselves are correct)
if strings.HasSuffix(f.Name, "/") {
continue
}
actualFiles = append(actualFiles, f.Name)
}
sort.Strings(expectedFiles)
sort.Strings(actualFiles)
assert.Equal(t, expectedFiles, actualFiles)
}
type fileContents struct {
name string
contents []byte
shouldRetain bool
}

40
pkg/util/archive/ignorer.go
View file

@ -1,40 +0,0 @@
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
type ignorer interface {
IsIgnored(f string) bool
}
type ignoreState struct {
ignorer ignorer
next *ignoreState
}
func (s *ignoreState) Append(ignorer ignorer) *ignoreState {
return &ignoreState{ignorer: ignorer, next: s}
}
func (s *ignoreState) IsIgnored(path string) bool {
if s == nil {
return false
}
if s.ignorer.IsIgnored(path) {
return true
}
return s.next.IsIgnored(path)
}

152
pkg/util/archive/ignorer_node_modules.go
View file

@ -1,152 +0,0 @@
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"container/list"
"encoding/json"
"os"
"path"
"strings"
"github.com/pulumi/pulumi/pkg/util/contract"
)
const nodeModulesDirName = "node_modules"
const packageJSONFileName = "package.json"
// newNodeModulesIgnorer constructs a new ignorer that ignores all files under node_modules that do not belong to
// packages in the transitive closure of the dependencies section of a package.json.
func newNodeModulesIgnorer(packageJSONPath string) (ignorer, error) {
depRoots, err := expandDepenencies(packageJSONPath)
if err != nil {
return nil, err
}
return &nodeModulesIgnorer{root: path.Dir(packageJSONPath), allowed: depRoots}, nil
}
func expandDepenencies(packageJSONPath string) ([]string, error) {
deps, err := getPackageDependencies(packageJSONPath)
if err != nil {
return nil, err
}
queue := list.New()
root := path.Dir(packageJSONPath)
for _, dep := range deps {
queue.PushBack(packageContext{name: dep, root: root})
}
var found struct{}
marked := make(map[string]struct{})
// While we still have packages we haven't processed, visit one.
for queue.Len() > 0 {
elm := queue.Front()
queue.Remove(elm)
ctx := elm.Value.(packageContext)
for curDir := ctx.root; strings.HasPrefix(curDir, root); curDir = path.Dir(curDir) {
candidateModuleFolder := path.Join(curDir, nodeModulesDirName, ctx.name)
if _, err := os.Stat(candidateModuleFolder); err == nil {
_, has := marked[candidateModuleFolder]
marked[candidateModuleFolder] = found
// If we hadn't already seen this module, ensure we also visit its dependencies.
if !has {
deps, err := getPackageDependencies(path.Join(candidateModuleFolder, packageJSONFileName))
if err != nil {
return nil, err
}
for _, dep := range deps {
queue.PushBack(packageContext{name: dep, root: candidateModuleFolder})
}
}
// Don't explore up the file system tree, since we found a match.
break
}
}
}
ret := make([]string, 0, len(marked))
for dir := range marked {
ret = append(ret, dir)
}
return ret, nil
}
func getPackageDependencies(packageJSONPath string) ([]string, error) {
f, err := os.Open(packageJSONPath)
if err != nil {
return nil, err
}
defer func() {
contract.IgnoreError(f.Close())
}()
dec := json.NewDecoder(f)
var file packageJSONFile
err = dec.Decode(&file)
if err != nil {
return nil, err
}
packages := make([]string, 0, len(file.Dependencies))
for dep := range file.Dependencies {
packages = append(packages, dep)
}
return packages, nil
}
type nodeModulesIgnorer struct {
root string // the directory for the node project this ingorer was built from
allowed []string // the list of all node_modules folders that the project needs
}
// MatchesPath for a nodeModulesIngorer can ignore any file under node_modules for the project
// it was constructed from as along as long as it is not in a directory for a module we know
// we must retain.
func (n *nodeModulesIgnorer) IsIgnored(f string) bool {
if !strings.HasPrefix(f, path.Join(n.root, nodeModulesDirName)) {
return false
}
for _, prefix := range n.allowed {
if strings.HasPrefix(f, prefix) {
return false
}
}
return true
}
type packageContext struct {
name string // The name of the package from the package.json.
root string // The directory of the package.json this was found in (this is where we have to start our search).
}
type packageJSONFile struct {
Dependencies map[string]string `json:"dependencies"`
}

46
pkg/util/archive/ignorer_path.go
View file

@ -1,46 +0,0 @@
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"os"
"strings"
)
// newPathIgnorer creates an ignorer based that ignores either a single file (when dir is false) or
// and entire directory tree (when dir is true).
func newPathIgnorer(path string, isDir bool) ignorer {
if !isDir {
return &fileIgnorer{path: path}
}
return &directoryIgnorer{path: path + string(os.PathSeparator)}
}
type fileIgnorer struct {
path string
}
func (fi *fileIgnorer) IsIgnored(f string) bool {
return f == fi.path
}
type directoryIgnorer struct {
path string
}
func (di *directoryIgnorer) IsIgnored(f string) bool {
return strings.HasPrefix(f, di.path)
}

42
pkg/util/archive/ignorer_pulumiignore.go
View file

@ -1,42 +0,0 @@
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package archive
import (
"path"
"strings"
ignore "github.com/sabhiram/go-gitignore"
)
// newPulumiIgnorerIgnorer creates an ignorer based on the contents of a .pulumiignore file, which
// has the same semantics as a .gitignore file
func newPulumiIgnorerIgnorer(pathToPulumiIgnore string) (ignorer, error) {
gitIgnorer, err := ignore.CompileIgnoreFile(pathToPulumiIgnore)
if err != nil {
return nil, err
}
return &pulumiIgnoreIgnorer{root: path.Dir(pathToPulumiIgnore), ignorer: gitIgnorer}, nil
}
type pulumiIgnoreIgnorer struct {
root string
ignorer *ignore.GitIgnore
}
func (g *pulumiIgnoreIgnorer) IsIgnored(f string) bool {
return g.ignorer.MatchesPath(strings.TrimPrefix(f, g.root))
}

48
pkg/workspace/plugins.go
View file

@ -15,8 +15,6 @@
package workspace
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
@ -31,6 +29,8 @@ import (
"sort"
"time"
"github.com/pulumi/pulumi/pkg/util/archive"
"github.com/blang/semver"
"github.com/djherbis/times"
"github.com/pkg/errors"
@ -238,49 +238,17 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error {
contract.IgnoreError(os.RemoveAll(tempDir))
}()
// Unzip and untar the file as we go. We do this inside a function so that the `defer`'s to close files happen
// before we later try to rename the directory. Otherwise, the open file handles cause issues on Windows.
// Uncompress the plugin. We do this inside a function so that the `defer`'s to close files
// happen before we later try to rename the directory. Otherwise, the open file handles cause
// issues on Windows.
err = (func() error {
defer contract.IgnoreClose(tarball)
gzr, err := gzip.NewReader(tarball)
tarballBytes, err := ioutil.ReadAll(tarball)
if err != nil {
return errors.Wrapf(err, "unzipping")
}
r := tar.NewReader(gzr)
for {
header, err := r.Next()
if err == io.EOF {
break
} else if err != nil {
return errors.Wrapf(err, "untarring")
}
path := filepath.Join(tempDir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
// Create any directories as needed.
if _, err := os.Stat(path); err != nil {
if err = os.MkdirAll(path, 0700); err != nil {
return errors.Wrapf(err, "untarring dir %s", path)
}
}
case tar.TypeReg:
// Expand files into the target directory.
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return errors.Wrapf(err, "opening file %s for untar", path)
}
defer contract.IgnoreClose(dst)
if _, err = io.Copy(dst, r); err != nil {
return errors.Wrapf(err, "untarring file %s", path)
}
default:
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
}
return err
}
return nil
return archive.Untgz(tarballBytes, tempDir)
})()
if err != nil {
return err