From 7070304f81a2cd97f20ec4d50ee1a4706fcda352 Mon Sep 17 00:00:00 2001 From: Alex Clemmer Date: Thu, 1 Aug 2019 11:05:57 -0700 Subject: [PATCH] Use `npm pack` for policy publish Fixes #35. --- cmd/new.go | 45 ++---- pkg/backend/httpstate/policypack.go | 65 ++++++-- pkg/npm/npm.go | 66 ++++++++ pkg/util/archive/archive.go | 196 ++++------------------- pkg/util/archive/archive_test.go | 128 --------------- pkg/util/archive/ignorer.go | 40 ----- pkg/util/archive/ignorer_node_modules.go | 152 ------------------ pkg/util/archive/ignorer_path.go | 46 ------ pkg/util/archive/ignorer_pulumiignore.go | 42 ----- pkg/workspace/plugins.go | 48 +----- 10 files changed, 173 insertions(+), 655 deletions(-) create mode 100644 pkg/npm/npm.go delete mode 100644 pkg/util/archive/archive_test.go delete mode 100644 pkg/util/archive/ignorer.go delete mode 100644 pkg/util/archive/ignorer_node_modules.go delete mode 100644 pkg/util/archive/ignorer_path.go delete mode 100644 pkg/util/archive/ignorer_pulumiignore.go diff --git a/cmd/new.go b/cmd/new.go index 2aad3e0e8..d3081eb41 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -18,33 +18,30 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path/filepath" "runtime" "sort" "strings" "unicode" - "github.com/pulumi/pulumi/pkg/backend/state" + "github.com/pkg/errors" + "github.com/spf13/cobra" + survey "gopkg.in/AlecAivazis/survey.v1" + surveycore "gopkg.in/AlecAivazis/survey.v1/core" "github.com/pulumi/pulumi/pkg/apitype" "github.com/pulumi/pulumi/pkg/backend" "github.com/pulumi/pulumi/pkg/backend/display" "github.com/pulumi/pulumi/pkg/backend/httpstate" + "github.com/pulumi/pulumi/pkg/backend/state" + "github.com/pulumi/pulumi/pkg/diag/colors" + "github.com/pulumi/pulumi/pkg/npm" "github.com/pulumi/pulumi/pkg/resource/config" "github.com/pulumi/pulumi/pkg/tokens" - "github.com/pulumi/pulumi/pkg/workspace" - - "github.com/pkg/errors" - "github.com/pulumi/pulumi/pkg/diag/colors" - "github.com/pulumi/pulumi/pkg/util/cmdutil" "github.com/pulumi/pulumi/pkg/util/contract" "github.com/pulumi/pulumi/pkg/util/logging" - "github.com/spf13/cobra" - - survey "gopkg.in/AlecAivazis/survey.v1" - surveycore "gopkg.in/AlecAivazis/survey.v1/core" + "github.com/pulumi/pulumi/pkg/workspace" ) // Intentionally disabling here for cleaner err declaration/assignment. @@ -466,15 +463,7 @@ func installDependencies() error { return err } - // TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here. - var command string - var c *exec.Cmd - if strings.EqualFold(proj.Runtime.Name(), "nodejs") { - command = "npm install" - // We pass `--loglevel=error` to prevent `npm` from printing warnings about missing - // `description`, `repository`, and `license` fields in the package.json file. - c = exec.Command("npm", "install", "--loglevel=error") - } else { + if !strings.EqualFold(proj.Runtime.Name(), "nodejs") { return nil } @@ -482,17 +471,11 @@ func installDependencies() error { fmt.Println() // Run the command. - c.Stdout = os.Stdout - c.Stderr = os.Stderr - if err := c.Run(); err != nil { - return errors.Wrapf(err, "installing dependencies; rerun '%s' manually to try again, "+ - "then run 'pulumi up' to perform an initial deployment", command) - } - - // Ensure the "node_modules" directory exists. - if _, err := os.Stat("node_modules"); os.IsNotExist(err) { - return errors.Errorf("installing dependencies; rerun '%s' manually to try again, "+ - "then run 'pulumi up' to perform an initial deployment", command) + // TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here. + err = npm.Install("", os.Stdout, os.Stderr) + if err != nil { + return errors.Wrapf(err, "npm install failed; rerun manually to try again, "+ + "then run 'pulumi up' to perform an initial deployment") } fmt.Println("Finished installing dependencies") diff --git a/pkg/backend/httpstate/policypack.go b/pkg/backend/httpstate/policypack.go index bf83c510e..7e0dc8296 100644 --- a/pkg/backend/httpstate/policypack.go +++ b/pkg/backend/httpstate/policypack.go @@ -1,12 +1,19 @@ package httpstate import ( + "bytes" "context" "fmt" "io/ioutil" "os" + "path" "path/filepath" "strconv" + "strings" + + "github.com/pulumi/pulumi/pkg/util/archive" + + "github.com/pulumi/pulumi/pkg/npm" "github.com/pkg/errors" "github.com/pulumi/pulumi/pkg/apitype" @@ -14,7 +21,6 @@ import ( "github.com/pulumi/pulumi/pkg/backend/httpstate/client" "github.com/pulumi/pulumi/pkg/engine" "github.com/pulumi/pulumi/pkg/tokens" - "github.com/pulumi/pulumi/pkg/util/archive" "github.com/pulumi/pulumi/pkg/util/contract" "github.com/pulumi/pulumi/pkg/util/result" "github.com/pulumi/pulumi/pkg/workspace" @@ -48,12 +54,12 @@ func (rp *cloudRequiredPolicy) Install(ctx context.Context) (string, error) { } // PolicyPack has not been downloaded and installed. Do this now. - policyPackZip, err := rp.client.DownloadPolicyPack(ctx, policy.PackLocation) + policyPackTarball, err := rp.client.DownloadPolicyPack(ctx, policy.PackLocation) if err != nil { return "", err } - return policyPackPath, installRequiredPolicy(policyPackPath, policyPackZip) + return policyPackPath, installRequiredPolicy(policyPackPath, policyPackTarball) } func newCloudBackendPolicyPackReference( @@ -127,9 +133,17 @@ func (pack *cloudPolicyPack) Publish( fmt.Println("Compressing policy pack") - dirArchive, err := archive.Process(op.Root, false) + if runtime := pack.b.currentProject.Runtime.Name(); !strings.EqualFold(runtime, "nodejs") { + return result.Errorf( + "failed to publish policies becuase Pulumi.yaml requests unsupported runtime %s", + runtime) + } + + // TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here. + packTarball, err := npm.Pack(op.PlugCtx.Pwd, os.Stderr) if err != nil { - return result.FromError(err) + return result.FromError( + errors.Wrapf(err, "could not publish policies because of error running npm pack")) } // @@ -138,7 +152,7 @@ func (pack *cloudPolicyPack) Publish( fmt.Println("Uploading policy pack to Pulumi service") - err = pack.cl.PublishPolicyPack(ctx, pack.ref.orgName, analyzerInfo, dirArchive) + err = pack.cl.PublishPolicyPack(ctx, pack.ref.orgName, analyzerInfo, bytes.NewReader(packTarball)) if err != nil { return result.FromError(err) } @@ -150,7 +164,9 @@ func (pack *cloudPolicyPack) Apply(ctx context.Context, op backend.ApplyOperatio return pack.cl.ApplyPolicyPack(ctx, pack.ref.orgName, string(pack.ref.name), op.Version) } -func installRequiredPolicy(finalDir string, zip []byte) error { +const npmPackageDir = "package" + +func installRequiredPolicy(finalDir string, tarball []byte) error { // If part of the directory tree is missing, ioutil.TempDir will return an error, so make sure // the path we're going to create the temporary folder in actually exists. if err := os.MkdirAll(filepath.Dir(finalDir), 0700); err != nil { @@ -162,14 +178,19 @@ func installRequiredPolicy(finalDir string, zip []byte) error { return errors.Wrapf(err, "creating plugin directory %s", tempDir) } + // npm unpacks into a directory called `package`. + tempNPMPkgDir := path.Join(tempDir, npmPackageDir) + if err := os.MkdirAll(tempNPMPkgDir, 0700); err != nil { + return errors.Wrap(err, "creating plugin root") + } + // If we early out of this function, try to remove the temp folder we created. defer func() { contract.IgnoreError(os.RemoveAll(tempDir)) }() - // Unzip the file. NOTE: It is important that the `Close` calls in this function complete before - // we try to rename the temp directory. Open file handles here cause issues on Windows. - err = archive.Unzip(zip, tempDir) + // Uncompress the policy pack. + err = archive.Untgz(tarball, tempDir) if err != nil { return err } @@ -179,9 +200,31 @@ func installRequiredPolicy(finalDir string, zip []byte) error { // If two calls to `plugin install` for the same plugin are racing, the second one will be // unable to rename the directory. That's OK, just ignore the error. The temp directory created // as part of the install will be cleaned up when we exit by the defer above. - if err := os.Rename(tempDir, finalDir); err != nil && !os.IsExist(err) { + if err := os.Rename(tempNPMPkgDir, finalDir); err != nil && !os.IsExist(err) { return errors.Wrap(err, "moving plugin") } + proj, err := workspace.LoadProject(path.Join(finalDir, "Pulumi.yaml")) + if err != nil { + return errors.Wrapf(err, "failed to load policy project at %s", finalDir) + } + + // TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here. + if !strings.EqualFold(proj.Runtime.Name(), "nodejs") { + return fmt.Errorf("unsupported policy runtime %s", proj.Runtime.Name()) + } + + fmt.Println("Installing dependencies...") + fmt.Println() + + // TODO[pulumi/pulumi#1307]: move to the language plugins so we don't have to hard code here. + err = npm.Install(finalDir, nil, nil) + if err != nil { + return err + } + + fmt.Println("Finished installing dependencies") + fmt.Println() + return nil } diff --git a/pkg/npm/npm.go b/pkg/npm/npm.go new file mode 100644 index 000000000..4539772e1 --- /dev/null +++ b/pkg/npm/npm.go @@ -0,0 +1,66 @@ +package npm + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "strings" + + "github.com/pkg/errors" +) + +// Pack runs `npm pack` in the given directory, packaging the Node.js app located there into a +// tarball and returning it as `[]byte`. `stdout` is ignored for the command, as it does not +// generate useful data. +func Pack(dir string, stderr io.Writer) ([]byte, error) { + // We pass `--loglevel=error` to prevent `npm` from printing warnings about missing + // `description`, `repository`, and `license` fields in the package.json file. + c := exec.Command("npm", "pack", "--loglevel=error") + c.Dir = dir + + // Run the command. Note that `npm pack` doesn't have the ability to rename the resulting + // filename, since it's meant to be uploaded directly to npm, which means that we have to get + // that information by parsing the output of the command. + var stdout bytes.Buffer + c.Stdout = &stdout + c.Stderr = os.Stderr + if err := c.Run(); err != nil { + return nil, err + } + packfile := strings.TrimSpace(stdout.String()) + defer os.Remove(packfile) + + packTarball, err := ioutil.ReadFile(packfile) + if err != nil { + return nil, fmt.Errorf( + "npm pack completed successfully but the packed .tar.gz file was not generated") + } + + return packTarball, nil +} + +// Install runs `npm install` in the given directory, installing the dependencies for the Node.js +// app located there. +func Install(dir string, stdout, stderr io.Writer) error { + // We pass `--loglevel=error` to prevent `npm` from printing warnings about missing + // `description`, `repository`, and `license` fields in the package.json file. + c := exec.Command("npm", "install", "--loglevel=error") + c.Dir = dir + + // Run the command. + c.Stdout = stdout + c.Stderr = stderr + if err := c.Run(); err != nil { + return err + } + + // Ensure the "node_modules" directory exists. + if _, err := os.Stat("node_modules"); os.IsNotExist(err) { + return errors.Errorf("npm install reported success, but node_modeules directory is missing") + } + + return nil +} diff --git a/pkg/util/archive/archive.go b/pkg/util/archive/archive.go index fd6c18de3..de799f56e 100644 --- a/pkg/util/archive/archive.go +++ b/pkg/util/archive/archive.go @@ -18,191 +18,57 @@ package archive import ( - "archive/zip" + "archive/tar" "bytes" + "compress/gzip" "io" "os" - "path" "path/filepath" - "sort" - "strings" "github.com/pkg/errors" - "github.com/pulumi/pulumi/pkg/util/contract" - "github.com/pulumi/pulumi/pkg/util/logging" - "github.com/pulumi/pulumi/pkg/workspace" ) -// Process returns an in-memory buffer with the archived contents of the provided file path. -func Process(path string, useDefaultExcludes bool) (*bytes.Buffer, error) { - buffer := &bytes.Buffer{} - writer := zip.NewWriter(buffer) - - // We trim `path` from the pathname of every file we add to the zip, but we actaually - // want to ensure the files directly under `path` are not added with a path prefix, - // so we add an extra os.PathSeparator here to the end of the string if it doesn't - // already end with one. - if !os.IsPathSeparator(path[len(path)-1]) { - path = path + string(os.PathSeparator) - } - - if err := addDirectoryToZip(writer, path, path, useDefaultExcludes, nil); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - - logging.V(5).Infof("project archive is %v bytes", buffer.Len()) - - return buffer, nil -} - -// Unzip a zip file into a specific directory. -func Unzip(archive []byte, dir string) error { - r, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) +// Untgz uncompresses a .tar.gz/.tgz file into a specific directory. +func Untgz(tarball []byte, dir string) error { + tarReader := bytes.NewReader(tarball) + gzr, err := gzip.NewReader(tarReader) if err != nil { - return err + return errors.Wrapf(err, "unzipping") } - - sort.Slice(r.File, func(i, j int) bool { return r.File[i].Name < r.File[j].Name }) - - for _, file := range r.File { - path := filepath.Join(dir, file.Name) // #nosec - if file.FileInfo().IsDir() { - contract.IgnoreError(os.MkdirAll(path, file.Mode())) - continue - } else { - contract.IgnoreError(os.MkdirAll(filepath.Dir(path), 0700)) + r := tar.NewReader(gzr) + for { + header, err := r.Next() + if err == io.EOF { + break + } else if err != nil { + return errors.Wrapf(err, "untarring") } - fileReader, err := file.Open() - if err != nil { - return errors.Wrapf(err, "Failed to open file") - } - defer fileReader.Close() + path := filepath.Join(dir, header.Name) - targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) - if err != nil { - return errors.Wrapf(err, "Failed to open target file") - } - defer targetFile.Close() - - if _, err := io.Copy(targetFile, fileReader); err != nil { - return err - } - - // Eagerly try to close to avoid having too many open file descriptors for big archives. We - // simply ignore double-close errors. - fileReader.Close() - targetFile.Close() - } - - return nil -} - -func addDirectoryToZip(writer *zip.Writer, root string, dir string, - useDefaultIgnores bool, ignores *ignoreState) error { - ignoreFilePath := path.Join(dir, workspace.IgnoreFile) - - // If there is an ignorefile, process it before looking at any child paths. - if stat, err := os.Stat(ignoreFilePath); err == nil && !stat.IsDir() { - logging.V(9).Infof("processing ignore file in %v", dir) - - ignore, err := newPulumiIgnorerIgnorer(ignoreFilePath) - if err != nil { - return errors.Wrapf(err, "could not read ignore file in %v", dir) - } - - ignores = ignores.Append(ignore) - } - - if useDefaultIgnores { - dotGitPath := path.Join(dir, ".git") - if stat, err := os.Stat(dotGitPath); err == nil { - ignores = ignores.Append(newPathIgnorer(dotGitPath, stat.IsDir())) - } - - // If there is a package.json file here, let's build a node_modules ignorer from it. - packageJSONFilePath := path.Join(dir, packageJSONFileName) - if stat, err := os.Stat(packageJSONFilePath); err == nil && !stat.IsDir() { - logging.V(9).Infof("building ignore filter from package.json in %v", dir) - ignore, err := newNodeModulesIgnorer(packageJSONFilePath) - if err != nil { - return errors.Wrapf(err, "could not read ignores from package.json file in %v", dir) + switch header.Typeflag { + case tar.TypeDir: + // Create any directories as needed. + if _, err := os.Stat(path); err != nil { + if err = os.MkdirAll(path, 0700); err != nil { + return errors.Wrapf(err, "untarring dir %s", path) + } } - - ignores = ignores.Append(ignore) - } - - } - - file, err := os.Open(dir) - if err != nil { - return err - } - // No defer because we want to close file as soon as possible (right after we call Readdir). - - infos, err := file.Readdir(-1) - contract.IgnoreClose(file) - if err != nil { - return err - } - - for _, info := range infos { - fullName := path.Join(dir, info.Name()) - - if !info.IsDir() && ignores.IsIgnored(fullName) { - logging.V(9).Infof("skip archiving of %v due to ignore file", fullName) - continue - } - - // Resolve symlinks (Readdir above calls os.Lstat which does not follow symlinks). - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - info, err = os.Stat(fullName) + case tar.TypeReg: + // Expand files into the target directory. + dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { - return err + return errors.Wrapf(err, "opening file %s for untar", path) } - } - - if info.Mode().IsDir() { - err = addDirectoryToZip(writer, root, fullName, useDefaultIgnores, ignores) - if err != nil { - return err + defer contract.IgnoreClose(dst) + if _, err = io.Copy(dst, r); err != nil { + return errors.Wrapf(err, "untarring file %s", path) } - } else if info.Mode().IsRegular() { - logging.V(9).Infof("adding %v to archive", fullName) - - w, err := writer.Create(convertPathsForZip(strings.TrimPrefix(fullName, root))) - if err != nil { - return err - } - - file, err := os.Open(fullName) - if err != nil { - return err - } - // no defer because we want to close file as soon as possible (right after we call Copy) - - _, err = io.Copy(w, file) - contract.IgnoreClose(file) - if err != nil { - return err - } - } else { - logging.V(9).Infof("ignoring special file %v with mode %v", fullName, info.Mode()) + default: + return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag) } } return nil } - -// convertPathsForZip ensures that '/' is uses at the path separator in zip files. -func convertPathsForZip(path string) string { - if os.PathSeparator != '/' { - return strings.Replace(path, string(os.PathSeparator), "/", -1) - } - - return path -} diff --git a/pkg/util/archive/archive_test.go b/pkg/util/archive/archive_test.go deleted file mode 100644 index 9062d0f2e..000000000 --- a/pkg/util/archive/archive_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package archive - -import ( - "archive/zip" - "bytes" - "fmt" - "io/ioutil" - "os" - "path" - "sort" - "strings" - "testing" - - "github.com/pulumi/pulumi/pkg/util/contract" - - "github.com/stretchr/testify/assert" -) - -func TestIngoreSimple(t *testing.T) { - doArchiveTest(t, - fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true}, - fileContents{name: "included.txt", shouldRetain: true}, - fileContents{name: "node_modules/included.txt", shouldRetain: true}, - fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false}, - fileContents{name: "node_modules/pulumi/excluded/excluded.txt", shouldRetain: false}) -} - -func TestIgnoreNegate(t *testing.T) { - doArchiveTest(t, - fileContents{name: ".pulumiignore", contents: []byte("/*\n!/foo\n/foo/*\n!/foo/bar"), shouldRetain: false}, - fileContents{name: "excluded.txt", shouldRetain: false}, - fileContents{name: "foo/excluded.txt", shouldRetain: false}, - fileContents{name: "foo/baz/exlcuded.txt", shouldRetain: false}, - fileContents{name: "foo/bar/included.txt", shouldRetain: true}) -} - -func TestNested(t *testing.T) { - doArchiveTest(t, - fileContents{name: ".pulumiignore", contents: []byte("node_modules/pulumi/"), shouldRetain: true}, - fileContents{name: "node_modules/.pulumiignore", contents: []byte("@pulumi/"), shouldRetain: true}, - fileContents{name: "included.txt", shouldRetain: true}, - fileContents{name: "node_modules/included.txt", shouldRetain: true}, - fileContents{name: "node_modules/pulumi/excluded.txt", shouldRetain: false}, - fileContents{name: "node_modules/@pulumi/pulumi-cloud/excluded.txt", shouldRetain: false}) -} - -func doArchiveTest(t *testing.T, files ...fileContents) { - archive, err := archiveContents(files...) - assert.NoError(t, err) - - fmt.Println(archive.Len()) - - r, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len())) - assert.NoError(t, err) - - checkFiles(t, files, r.File) -} - -func archiveContents(files ...fileContents) (*bytes.Buffer, error) { - dir, err := ioutil.TempDir("", "archive-test") - if err != nil { - return nil, err - } - - defer func() { - contract.IgnoreError(os.RemoveAll(dir)) - }() - - for _, file := range files { - err := os.MkdirAll(path.Dir(path.Join(dir, file.name)), 0755) - if err != nil { - return nil, err - } - - err = ioutil.WriteFile(path.Join(dir, file.name), file.contents, 0644) - if err != nil { - return nil, err - } - } - - return Process(dir, false) -} - -func checkFiles(t *testing.T, expected []fileContents, actual []*zip.File) { - var expectedFiles []string - var actualFiles []string - - for _, f := range expected { - if f.shouldRetain { - expectedFiles = append(expectedFiles, f.name) - } - } - - for _, f := range actual { - - // Ignore any directories (we only care that the files themselves are correct) - if strings.HasSuffix(f.Name, "/") { - continue - } - - actualFiles = append(actualFiles, f.Name) - } - - sort.Strings(expectedFiles) - sort.Strings(actualFiles) - - assert.Equal(t, expectedFiles, actualFiles) -} - -type fileContents struct { - name string - contents []byte - shouldRetain bool -} diff --git a/pkg/util/archive/ignorer.go b/pkg/util/archive/ignorer.go deleted file mode 100644 index 06cda57a7..000000000 --- a/pkg/util/archive/ignorer.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package archive - -type ignorer interface { - IsIgnored(f string) bool -} - -type ignoreState struct { - ignorer ignorer - next *ignoreState -} - -func (s *ignoreState) Append(ignorer ignorer) *ignoreState { - return &ignoreState{ignorer: ignorer, next: s} -} - -func (s *ignoreState) IsIgnored(path string) bool { - if s == nil { - return false - } - - if s.ignorer.IsIgnored(path) { - return true - } - - return s.next.IsIgnored(path) -} diff --git a/pkg/util/archive/ignorer_node_modules.go b/pkg/util/archive/ignorer_node_modules.go deleted file mode 100644 index e05f4d064..000000000 --- a/pkg/util/archive/ignorer_node_modules.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package archive - -import ( - "container/list" - "encoding/json" - "os" - "path" - "strings" - - "github.com/pulumi/pulumi/pkg/util/contract" -) - -const nodeModulesDirName = "node_modules" -const packageJSONFileName = "package.json" - -// newNodeModulesIgnorer constructs a new ignorer that ignores all files under node_modules that do not belong to -// packages in the transitive closure of the dependencies section of a package.json. -func newNodeModulesIgnorer(packageJSONPath string) (ignorer, error) { - - depRoots, err := expandDepenencies(packageJSONPath) - if err != nil { - return nil, err - } - - return &nodeModulesIgnorer{root: path.Dir(packageJSONPath), allowed: depRoots}, nil -} - -func expandDepenencies(packageJSONPath string) ([]string, error) { - deps, err := getPackageDependencies(packageJSONPath) - if err != nil { - return nil, err - } - - queue := list.New() - root := path.Dir(packageJSONPath) - for _, dep := range deps { - queue.PushBack(packageContext{name: dep, root: root}) - } - - var found struct{} - marked := make(map[string]struct{}) - - // While we still have packages we haven't processed, visit one. - for queue.Len() > 0 { - elm := queue.Front() - queue.Remove(elm) - - ctx := elm.Value.(packageContext) - - for curDir := ctx.root; strings.HasPrefix(curDir, root); curDir = path.Dir(curDir) { - candidateModuleFolder := path.Join(curDir, nodeModulesDirName, ctx.name) - - if _, err := os.Stat(candidateModuleFolder); err == nil { - _, has := marked[candidateModuleFolder] - marked[candidateModuleFolder] = found - - // If we hadn't already seen this module, ensure we also visit its dependencies. - if !has { - deps, err := getPackageDependencies(path.Join(candidateModuleFolder, packageJSONFileName)) - if err != nil { - return nil, err - } - - for _, dep := range deps { - queue.PushBack(packageContext{name: dep, root: candidateModuleFolder}) - } - } - - // Don't explore up the file system tree, since we found a match. - break - } - } - } - - ret := make([]string, 0, len(marked)) - for dir := range marked { - ret = append(ret, dir) - } - - return ret, nil -} - -func getPackageDependencies(packageJSONPath string) ([]string, error) { - - f, err := os.Open(packageJSONPath) - if err != nil { - return nil, err - } - defer func() { - contract.IgnoreError(f.Close()) - }() - - dec := json.NewDecoder(f) - var file packageJSONFile - - err = dec.Decode(&file) - if err != nil { - return nil, err - } - - packages := make([]string, 0, len(file.Dependencies)) - for dep := range file.Dependencies { - packages = append(packages, dep) - } - - return packages, nil -} - -type nodeModulesIgnorer struct { - root string // the directory for the node project this ingorer was built from - allowed []string // the list of all node_modules folders that the project needs -} - -// MatchesPath for a nodeModulesIngorer can ignore any file under node_modules for the project -// it was constructed from as along as long as it is not in a directory for a module we know -// we must retain. -func (n *nodeModulesIgnorer) IsIgnored(f string) bool { - if !strings.HasPrefix(f, path.Join(n.root, nodeModulesDirName)) { - return false - } - - for _, prefix := range n.allowed { - if strings.HasPrefix(f, prefix) { - return false - } - } - - return true -} - -type packageContext struct { - name string // The name of the package from the package.json. - root string // The directory of the package.json this was found in (this is where we have to start our search). -} - -type packageJSONFile struct { - Dependencies map[string]string `json:"dependencies"` -} diff --git a/pkg/util/archive/ignorer_path.go b/pkg/util/archive/ignorer_path.go deleted file mode 100644 index a7158bc62..000000000 --- a/pkg/util/archive/ignorer_path.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package archive - -import ( - "os" - "strings" -) - -// newPathIgnorer creates an ignorer based that ignores either a single file (when dir is false) or -// and entire directory tree (when dir is true). -func newPathIgnorer(path string, isDir bool) ignorer { - if !isDir { - return &fileIgnorer{path: path} - } - - return &directoryIgnorer{path: path + string(os.PathSeparator)} -} - -type fileIgnorer struct { - path string -} - -func (fi *fileIgnorer) IsIgnored(f string) bool { - return f == fi.path -} - -type directoryIgnorer struct { - path string -} - -func (di *directoryIgnorer) IsIgnored(f string) bool { - return strings.HasPrefix(f, di.path) -} diff --git a/pkg/util/archive/ignorer_pulumiignore.go b/pkg/util/archive/ignorer_pulumiignore.go deleted file mode 100644 index 0b8a583b0..000000000 --- a/pkg/util/archive/ignorer_pulumiignore.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2016-2018, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package archive - -import ( - "path" - "strings" - - ignore "github.com/sabhiram/go-gitignore" -) - -// newPulumiIgnorerIgnorer creates an ignorer based on the contents of a .pulumiignore file, which -// has the same semantics as a .gitignore file -func newPulumiIgnorerIgnorer(pathToPulumiIgnore string) (ignorer, error) { - gitIgnorer, err := ignore.CompileIgnoreFile(pathToPulumiIgnore) - if err != nil { - return nil, err - } - - return &pulumiIgnoreIgnorer{root: path.Dir(pathToPulumiIgnore), ignorer: gitIgnorer}, nil -} - -type pulumiIgnoreIgnorer struct { - root string - ignorer *ignore.GitIgnore -} - -func (g *pulumiIgnoreIgnorer) IsIgnored(f string) bool { - return g.ignorer.MatchesPath(strings.TrimPrefix(f, g.root)) -} diff --git a/pkg/workspace/plugins.go b/pkg/workspace/plugins.go index 60a3af561..0ddd50ad7 100644 --- a/pkg/workspace/plugins.go +++ b/pkg/workspace/plugins.go @@ -15,8 +15,6 @@ package workspace import ( - "archive/tar" - "compress/gzip" "fmt" "io" "io/ioutil" @@ -31,6 +29,8 @@ import ( "sort" "time" + "github.com/pulumi/pulumi/pkg/util/archive" + "github.com/blang/semver" "github.com/djherbis/times" "github.com/pkg/errors" @@ -238,49 +238,17 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error { contract.IgnoreError(os.RemoveAll(tempDir)) }() - // Unzip and untar the file as we go. We do this inside a function so that the `defer`'s to close files happen - // before we later try to rename the directory. Otherwise, the open file handles cause issues on Windows. + // Uncompress the plugin. We do this inside a function so that the `defer`'s to close files + // happen before we later try to rename the directory. Otherwise, the open file handles cause + // issues on Windows. err = (func() error { defer contract.IgnoreClose(tarball) - gzr, err := gzip.NewReader(tarball) + tarballBytes, err := ioutil.ReadAll(tarball) if err != nil { - return errors.Wrapf(err, "unzipping") - } - r := tar.NewReader(gzr) - for { - header, err := r.Next() - if err == io.EOF { - break - } else if err != nil { - return errors.Wrapf(err, "untarring") - } - - path := filepath.Join(tempDir, header.Name) - - switch header.Typeflag { - case tar.TypeDir: - // Create any directories as needed. - if _, err := os.Stat(path); err != nil { - if err = os.MkdirAll(path, 0700); err != nil { - return errors.Wrapf(err, "untarring dir %s", path) - } - } - case tar.TypeReg: - // Expand files into the target directory. - dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) - if err != nil { - return errors.Wrapf(err, "opening file %s for untar", path) - } - defer contract.IgnoreClose(dst) - if _, err = io.Copy(dst, r); err != nil { - return errors.Wrapf(err, "untarring file %s", path) - } - default: - return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag) - } + return err } - return nil + return archive.Untgz(tarballBytes, tempDir) })() if err != nil { return err