maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Do not read TGZs into memory. (#5983)

Browse source 
* Do not read TGZs into memory.

This runs a serious risk of exhausting the memory on lower-end machines
(e.g. certain CI VMs), especially given the potential size of some
plugins.

* CHANGELOG

* fixes
This commit is contained in:
Pat Gavlin 2020-12-20 12:54:11 -08:00 committed by GitHub
parent 8a9b381767
commit eeff5257c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 71 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -3,6 +3,10 @@ CHANGELOG
## HEAD (Unreleased)
- Do not read plugins and policy packs into memory prior to exctraction, as doing so can exhaust
the available memory on lower-end systems.
[#5983](https://github.com/pulumi/pulumi/pull/5983)
- Fix a bug in the core engine where deleting/renaming a resource would panic on update + refresh.
[#5980](https://github.com/pulumi/pulumi/pull/5980)

11
pkg/backend/httpstate/client/client.go
View file

@ -19,7 +19,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"path"
"regexp"
@ -753,7 +752,7 @@ func (pc *Client) RemovePolicyPackByVersion(ctx context.Context, orgName string,
}
// DownloadPolicyPack applies a `PolicyPack` to the Pulumi organization.
func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) ([]byte, error) {
func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) (io.ReadCloser, error) {
getS3Req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
@ -763,14 +762,8 @@ func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) ([]byte, e
if err != nil {
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
}
defer resp.Body.Close()
tarball, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
}
return tarball, nil
return resp.Body, nil
}
// GetUpdateEvents returns all events, taking an optional continuation token from a previous call.

5
pkg/backend/httpstate/policypack.go
View file

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -248,7 +249,7 @@ func (pack *cloudPolicyPack) Remove(ctx context.Context, op backend.PolicyPackOp
const packageDir = "package"
func installRequiredPolicy(finalDir string, tarball []byte) error {
func installRequiredPolicy(finalDir string, tgz io.ReadCloser) error {
// If part of the directory tree is missing, ioutil.TempDir will return an error, so make sure
// the path we're going to create the temporary folder in actually exists.
if err := os.MkdirAll(filepath.Dir(finalDir), 0700); err != nil {
@ -272,7 +273,7 @@ func installRequiredPolicy(finalDir string, tarball []byte) error {
}()
// Uncompress the policy pack.
err = archive.UnTGZ(tarball, tempDir)
err = archive.ExtractTGZ(tgz, tempDir)
if err != nil {
return err
}

110
sdk/go/common/util/archive/archive.go
View file

@ -60,66 +60,72 @@ func TGZ(dir, prefixPathInsideTar string, useDefaultExcludes bool) ([]byte, erro
return buffer.Bytes(), nil
}
// UnTGZ uncompresses a .tar.gz/.tgz file into a specific directory.
func UnTGZ(tarball []byte, dir string) error {
tarReader := bytes.NewReader(tarball)
gzr, err := gzip.NewReader(tarReader)
if err != nil {
return errors.Wrapf(err, "unzipping")
}
r := tar.NewReader(gzr)
for {
header, err := r.Next()
if err == io.EOF {
break
} else if err != nil {
return errors.Wrapf(err, "untarring")
func extractFile(r *tar.Reader, header *tar.Header, dir string) error {
// TODO: check the name to ensure that it does not contain path traversal characters.
//
//nolint: gosec
path := filepath.Join(dir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
// Create any directories as needed.
if _, err := os.Stat(path); err != nil {
if err = os.MkdirAll(path, 0700); err != nil {
return errors.Wrapf(err, "extracting dir %s", path)
}
}
case tar.TypeReg:
// Create any directories as needed. Some tools (notably `npm pack`) don't list
// directories individually, so if a file is in a directory that doesn't exist, we need
// to create it here.
dir := filepath.Dir(path)
if _, err := os.Stat(dir); err != nil {
if err = os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "extracting dir %s", dir)
}
}
// TODO: check the name to ensure that it does not contain path traversal characters.
//
//nolint: gosec
path := filepath.Join(dir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
// Create any directories as needed.
if _, err := os.Stat(path); err != nil {
if err = os.MkdirAll(path, 0700); err != nil {
return errors.Wrapf(err, "untarring dir %s", path)
}
}
case tar.TypeReg:
// Create any directories as needed. Some tools (notably `npm pack`) don't list
// directories individually, so if a file is in a directory that doesn't exist, we need
// to create it here.
dir := filepath.Dir(path)
if _, err := os.Stat(dir); err != nil {
if err = os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "untarring dir %s", dir)
}
}
// Expand files into the target directory.
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return errors.Wrapf(err, "opening file %s for untar", path)
}
defer contract.IgnoreClose(dst)
// We're not concerned with potential tarbombs, so disable gosec.
// nolint:gosec
if _, err = io.Copy(dst, r); err != nil {
return errors.Wrapf(err, "untarring file %s", path)
}
default:
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
// Expand files into the target directory.
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return errors.Wrapf(err, "opening file %s for extraction", path)
}
defer contract.IgnoreClose(dst)
// We're not concerned with potential tarbombs, so disable gosec.
// nolint:gosec
if _, err = io.Copy(dst, r); err != nil {
return errors.Wrapf(err, "untarring file %s", path)
}
default:
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
}
return nil
}
// ExtractTGZ uncompresses a .tar.gz/.tgz file into a specific directory.
func ExtractTGZ(r io.Reader, dir string) error {
gzr, err := gzip.NewReader(r)
if err != nil {
return errors.Wrapf(err, "uncompressing")
}
tr := tar.NewReader(gzr)
for {
header, err := tr.Next()
if err != nil {
if err == io.EOF {
return nil
}
return errors.Wrapf(err, "extracting")
}
if err = extractFile(tr, header, dir); err != nil {
return err
}
}
}
const (
gitDir = ".git"
gitIgnoreFile = ".gitignore"

12
sdk/go/common/workspace/plugins.go
View file

@ -298,8 +298,8 @@ func (info PluginInfo) installLock() (unlock func(), err error) {
// If a failure occurs during installation, the `.partial` file will remain, indicating the plugin wasn't fully
// installed. The next time the plugin is installed, the old installation directory will be removed and replaced with
// a fresh install.
func (info PluginInfo) Install(tarball io.ReadCloser) error {
defer contract.IgnoreClose(tarball)
func (info PluginInfo) Install(tgz io.ReadCloser) error {
defer contract.IgnoreClose(tgz)
// Fetch the directory into which we will expand this tarball.
finalDir, err := info.DirPath()
@ -360,18 +360,14 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error {
}
// Uncompress the plugin.
tarballBytes, err := ioutil.ReadAll(tarball)
if err != nil {
return err
}
if err := archive.UnTGZ(tarballBytes, finalDir); err != nil {
if err := archive.ExtractTGZ(tgz, finalDir); err != nil {
return err
}
// Even though we deferred closing the tarball at the beginning of this function, go ahead and explicitly close
// it now since we're finished extracting it, to prevent subsequent output from being displayed oddly with
// the progress bar.
contract.IgnoreClose(tarball)
contract.IgnoreClose(tgz)
// Install dependencies, if needed.
proj, err := LoadPluginProject(filepath.Join(finalDir, "PulumiPlugin.yaml"))