Do not read TGZs into memory. (#5983)
* Do not read TGZs into memory. This runs a serious risk of exhausting the memory on lower-end machines (e.g. certain CI VMs), especially given the potential size of some plugins. * CHANGELOG * fixes
This commit is contained in:
parent
8a9b381767
commit
eeff5257c3
|
@ -3,6 +3,10 @@ CHANGELOG
|
|||
|
||||
## HEAD (Unreleased)
|
||||
|
||||
- Do not read plugins and policy packs into memory prior to exctraction, as doing so can exhaust
|
||||
the available memory on lower-end systems.
|
||||
[#5983](https://github.com/pulumi/pulumi/pull/5983)
|
||||
|
||||
- Fix a bug in the core engine where deleting/renaming a resource would panic on update + refresh.
|
||||
[#5980](https://github.com/pulumi/pulumi/pull/5980)
|
||||
|
||||
|
|
|
@ -19,7 +19,6 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
|
@ -753,7 +752,7 @@ func (pc *Client) RemovePolicyPackByVersion(ctx context.Context, orgName string,
|
|||
}
|
||||
|
||||
// DownloadPolicyPack applies a `PolicyPack` to the Pulumi organization.
|
||||
func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) ([]byte, error) {
|
||||
func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) (io.ReadCloser, error) {
|
||||
getS3Req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
|
||||
|
@ -763,14 +762,8 @@ func (pc *Client) DownloadPolicyPack(ctx context.Context, url string) ([]byte, e
|
|||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
tarball, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to download compressed PolicyPack")
|
||||
}
|
||||
|
||||
return tarball, nil
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// GetUpdateEvents returns all events, taking an optional continuation token from a previous call.
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
@ -248,7 +249,7 @@ func (pack *cloudPolicyPack) Remove(ctx context.Context, op backend.PolicyPackOp
|
|||
|
||||
const packageDir = "package"
|
||||
|
||||
func installRequiredPolicy(finalDir string, tarball []byte) error {
|
||||
func installRequiredPolicy(finalDir string, tgz io.ReadCloser) error {
|
||||
// If part of the directory tree is missing, ioutil.TempDir will return an error, so make sure
|
||||
// the path we're going to create the temporary folder in actually exists.
|
||||
if err := os.MkdirAll(filepath.Dir(finalDir), 0700); err != nil {
|
||||
|
@ -272,7 +273,7 @@ func installRequiredPolicy(finalDir string, tarball []byte) error {
|
|||
}()
|
||||
|
||||
// Uncompress the policy pack.
|
||||
err = archive.UnTGZ(tarball, tempDir)
|
||||
err = archive.ExtractTGZ(tgz, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -60,66 +60,72 @@ func TGZ(dir, prefixPathInsideTar string, useDefaultExcludes bool) ([]byte, erro
|
|||
return buffer.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnTGZ uncompresses a .tar.gz/.tgz file into a specific directory.
|
||||
func UnTGZ(tarball []byte, dir string) error {
|
||||
tarReader := bytes.NewReader(tarball)
|
||||
gzr, err := gzip.NewReader(tarReader)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unzipping")
|
||||
}
|
||||
r := tar.NewReader(gzr)
|
||||
for {
|
||||
header, err := r.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return errors.Wrapf(err, "untarring")
|
||||
func extractFile(r *tar.Reader, header *tar.Header, dir string) error {
|
||||
// TODO: check the name to ensure that it does not contain path traversal characters.
|
||||
//
|
||||
//nolint: gosec
|
||||
path := filepath.Join(dir, header.Name)
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
// Create any directories as needed.
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if err = os.MkdirAll(path, 0700); err != nil {
|
||||
return errors.Wrapf(err, "extracting dir %s", path)
|
||||
}
|
||||
}
|
||||
case tar.TypeReg:
|
||||
// Create any directories as needed. Some tools (notably `npm pack`) don't list
|
||||
// directories individually, so if a file is in a directory that doesn't exist, we need
|
||||
// to create it here.
|
||||
dir := filepath.Dir(path)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
if err = os.MkdirAll(dir, 0700); err != nil {
|
||||
return errors.Wrapf(err, "extracting dir %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: check the name to ensure that it does not contain path traversal characters.
|
||||
//
|
||||
//nolint: gosec
|
||||
path := filepath.Join(dir, header.Name)
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
// Create any directories as needed.
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if err = os.MkdirAll(path, 0700); err != nil {
|
||||
return errors.Wrapf(err, "untarring dir %s", path)
|
||||
}
|
||||
}
|
||||
case tar.TypeReg:
|
||||
// Create any directories as needed. Some tools (notably `npm pack`) don't list
|
||||
// directories individually, so if a file is in a directory that doesn't exist, we need
|
||||
// to create it here.
|
||||
dir := filepath.Dir(path)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
if err = os.MkdirAll(dir, 0700); err != nil {
|
||||
return errors.Wrapf(err, "untarring dir %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// Expand files into the target directory.
|
||||
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "opening file %s for untar", path)
|
||||
}
|
||||
defer contract.IgnoreClose(dst)
|
||||
|
||||
// We're not concerned with potential tarbombs, so disable gosec.
|
||||
// nolint:gosec
|
||||
if _, err = io.Copy(dst, r); err != nil {
|
||||
return errors.Wrapf(err, "untarring file %s", path)
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
|
||||
// Expand files into the target directory.
|
||||
dst, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "opening file %s for extraction", path)
|
||||
}
|
||||
defer contract.IgnoreClose(dst)
|
||||
|
||||
// We're not concerned with potential tarbombs, so disable gosec.
|
||||
// nolint:gosec
|
||||
if _, err = io.Copy(dst, r); err != nil {
|
||||
return errors.Wrapf(err, "untarring file %s", path)
|
||||
}
|
||||
default:
|
||||
return errors.Errorf("unexpected plugin file type %s (%v)", header.Name, header.Typeflag)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractTGZ uncompresses a .tar.gz/.tgz file into a specific directory.
|
||||
func ExtractTGZ(r io.Reader, dir string) error {
|
||||
gzr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "uncompressing")
|
||||
}
|
||||
tr := tar.NewReader(gzr)
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "extracting")
|
||||
}
|
||||
|
||||
if err = extractFile(tr, header, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
gitDir = ".git"
|
||||
gitIgnoreFile = ".gitignore"
|
||||
|
|
|
@ -298,8 +298,8 @@ func (info PluginInfo) installLock() (unlock func(), err error) {
|
|||
// If a failure occurs during installation, the `.partial` file will remain, indicating the plugin wasn't fully
|
||||
// installed. The next time the plugin is installed, the old installation directory will be removed and replaced with
|
||||
// a fresh install.
|
||||
func (info PluginInfo) Install(tarball io.ReadCloser) error {
|
||||
defer contract.IgnoreClose(tarball)
|
||||
func (info PluginInfo) Install(tgz io.ReadCloser) error {
|
||||
defer contract.IgnoreClose(tgz)
|
||||
|
||||
// Fetch the directory into which we will expand this tarball.
|
||||
finalDir, err := info.DirPath()
|
||||
|
@ -360,18 +360,14 @@ func (info PluginInfo) Install(tarball io.ReadCloser) error {
|
|||
}
|
||||
|
||||
// Uncompress the plugin.
|
||||
tarballBytes, err := ioutil.ReadAll(tarball)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := archive.UnTGZ(tarballBytes, finalDir); err != nil {
|
||||
if err := archive.ExtractTGZ(tgz, finalDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Even though we deferred closing the tarball at the beginning of this function, go ahead and explicitly close
|
||||
// it now since we're finished extracting it, to prevent subsequent output from being displayed oddly with
|
||||
// the progress bar.
|
||||
contract.IgnoreClose(tarball)
|
||||
contract.IgnoreClose(tgz)
|
||||
|
||||
// Install dependencies, if needed.
|
||||
proj, err := LoadPluginProject(filepath.Join(finalDir, "PulumiPlugin.yaml"))
|
||||
|
|
Loading…
Reference in a new issue