maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move program uploads to the CLI (#571)

Browse source 
In an effort to improve performance and overall reliability, this PR moves the responsibility of uploading the Pulumi program from the Pulumi Service to the CLI. (Part of fixing https://github.com/pulumi/pulumi-service/issues/313.)

Previously the CLI would send (the dozens of MiB) program archive to the Service, which would then upload the data to S3. Now the CLI sends the data to S3 directly, avoiding the unnecessary copying of data around.

The Service-side API changes are in https://github.com/pulumi/pulumi-service/pull/323. I tested previews, updates, and destroys running the service and PPC on localhost.

The PR refactors how we handle the three kinds of program updates, and just unifies them into a single method. This makes the diff look crazy, but the code should be much simpler. I'm not sure what to do about supporting all the engine options for the Cloud-variants of Pulumi commands; I suspect that's something that should be handled at a later time.
This commit is contained in:
Chris Smith 2017-11-15 13:27:28 -08:00 committed by GitHub
parent 1a1c44b71e
commit 84cd810112
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 164 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -4,3 +4,5 @@
coverage.cov
*.coverprofile
# Too much awesome to store in Git.
.pulumi

207
cmd/backend_cloud.go
View file

@ -4,6 +4,9 @@ package cmd
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
@ -17,6 +20,7 @@ import (
"github.com/pulumi/pulumi/pkg/pack"
"github.com/pulumi/pulumi/pkg/resource/config"
"github.com/pulumi/pulumi/pkg/util/archive"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/workspace"
"github.com/pulumi/pulumi/pkg/tokens"
@ -87,75 +91,30 @@ func (b *pulumiCloudPulumiBackend) RemoveStack(stackName tokens.QName, force boo
return pulumiRESTCall("DELETE", path, nil, nil)
}
func (b *pulumiCloudPulumiBackend) Preview(stackName tokens.QName, debug bool, opts engine.PreviewOptions) error {
projID, err := getCloudProjectIdentifier()
if err != nil {
return err
}
updateRequest, err := makeProgramUpdateRequest(stackName)
if err != nil {
return err
}
// updateKind is an enum for describing the kinds of updates we support.
type updateKind = int
var response apitype.PreviewUpdateResponse
path := fmt.Sprintf("/orgs/%s/programs/%s/%s/stacks/%s/preview",
projID.Owner, projID.Repository, projID.Project, string(stackName))
if err = pulumiRESTCall("POST", path, &updateRequest, &response); err != nil {
return err
}
fmt.Printf("Previewing update to Stack '%s'...\n", string(stackName))
const (
update updateKind = iota
preview
destroy
)
// Wait for the update to complete.
status, err := waitForUpdate(fmt.Sprintf("%s/%s", path, response.UpdateID))
fmt.Println() // The PPC's final message we print to STDOUT doesn't include a newline.
if err != nil {
return errors.Errorf("waiting for preview: %v", err)
}
if status == apitype.StatusSucceeded {
fmt.Println("Preview resulted in success.")
return nil
}
return errors.Errorf("preview result was unsuccessful: status %v", status)
func (b *pulumiCloudPulumiBackend) Preview(stackName tokens.QName, debug bool, _ engine.PreviewOptions) error {
return updateStack(preview, stackName, debug)
}
func (b *pulumiCloudPulumiBackend) Update(stackName tokens.QName, debug bool, opts engine.DeployOptions) error {
projID, err := getCloudProjectIdentifier()
if err != nil {
return err
}
updateRequest, err := makeProgramUpdateRequest(stackName)
if err != nil {
return err
}
var response apitype.UpdateProgramResponse
path := fmt.Sprintf("/orgs/%s/programs/%s/%s/stacks/%s/update", projID.Owner, projID.Repository, projID.Project, string(stackName))
if err = pulumiRESTCall("POST", path, &updateRequest, &response); err != nil {
return err
}
fmt.Printf("Updating Stack '%s' to version %d...\n", string(stackName), response.Version)
// Wait for the update to complete.
status, err := waitForUpdate(fmt.Sprintf("%s/%s", path, response.UpdateID))
fmt.Println() // The PPC's final message we print to STDOUT doesn't include a newline.
if err != nil {
return errors.Errorf("waiting for update: %v", err)
}
if status == apitype.StatusSucceeded {
fmt.Println("Update completed successfully.")
return nil
}
return errors.Errorf("update unsuccessful: status %v", status)
func (b *pulumiCloudPulumiBackend) Update(stackName tokens.QName, debug bool, _ engine.DeployOptions) error {
return updateStack(update, stackName, debug)
}
func (b *pulumiCloudPulumiBackend) Destroy(stackName tokens.QName, debug bool, opts engine.DestroyOptions) error {
// TODO[pulumi/pulumi#516]: Once pulumi.com supports previews of destroys, remove this code
if opts.DryRun {
return errors.New("Pulumi.com does not support previewing destroy operations yet")
}
func (b *pulumiCloudPulumiBackend) Destroy(stackName tokens.QName, debug bool, _ engine.DestroyOptions) error {
return updateStack(destroy, stackName, debug)
}
// updateStack performs a the provided type of update on a stack hosted in the Pulumi Cloud.
func updateStack(kind updateKind, stackName tokens.QName, debug bool) error {
// First create the update object.
projID, err := getCloudProjectIdentifier()
if err != nil {
return err
@ -165,25 +124,105 @@ func (b *pulumiCloudPulumiBackend) Destroy(stackName tokens.QName, debug bool, o
return err
}
var response apitype.DestroyProgramResponse
path := fmt.Sprintf("/orgs/%s/programs/%s/%s/stacks/%s/destroy", projID.Owner, projID.Repository, projID.Project, string(stackName))
if err = pulumiRESTCall("POST", path, &updateRequest, &response); err != nil {
// Generate the URL we'll use for all the REST calls.
var action string
switch kind {
case update:
action = "update"
case preview:
action = "preview"
case destroy:
action = "destroy"
default:
contract.Failf("unsupported update kind: %v", kind)
}
restURLRoot := fmt.Sprintf(
"/orgs/%s/programs/%s/%s/stacks/%s/%s",
projID.Owner, projID.Repository, projID.Project, string(stackName), action)
// Create the initial update object.
var updateResponse apitype.UpdateProgramResponse
if err = pulumiRESTCall("POST", restURLRoot, &updateRequest, &updateResponse); err != nil {
return err
}
fmt.Printf("Destroying Stack '%s'...\n", string(stackName))
// Wait for the update to complete.
status, err := waitForUpdate(fmt.Sprintf("%s/%s", path, response.UpdateID))
// Upload the program's contents to the signed URL if appropriate.
if kind != destroy {
err = uploadProgram(updateResponse.UploadURL, debug /* print upload size to STDOUT */)
if err != nil {
return err
}
}
// Start the update.
restURLWithUpdateID := fmt.Sprintf("%s/%s", restURLRoot, updateResponse.UpdateID)
var startUpdateResponse apitype.StartUpdateResponse
if err = pulumiRESTCall("POST", restURLWithUpdateID, nil /* no req body */, &startUpdateResponse); err != nil {
return err
}
if kind == update {
fmt.Printf("Updating Stack '%s' to version %d...\n", string(stackName), startUpdateResponse.Version)
}
// Wait for the update to complete, which also polls and renders event output to STDOUT.
status, err := waitForUpdate(restURLWithUpdateID)
fmt.Println() // The PPC's final message we print to STDOUT doesn't include a newline.
if err != nil {
return errors.Errorf("waiting for destroy: %v", err)
return errors.Wrapf(err, "waiting for %s", action)
}
if status == apitype.StatusSucceeded {
fmt.Println("destroy complete.")
return nil
if status != apitype.StatusSucceeded {
return errors.Errorf("%s unsuccessful: status %v", action, status)
}
return errors.Errorf("destroy unsuccessful: status %v", status)
fmt.Printf("%s completed successfully.\n", action)
return nil
}
// uploadProgram archives the current Pulumi program and uploads it to a signed URL. "current"
// meaning whatever Pulumi program is found in the CWD or parent directory.
// If set, printSize will print the size of the data being uploaded.
func uploadProgram(uploadURL string, printSize bool) error {
cwd, err := os.Getwd()
if err != nil {
return errors.Wrap(err, "getting working directory")
}
programPath, err := workspace.DetectPackage(cwd)
if err != nil {
return errors.Wrap(err, "looking for Pulumi package")
}
if programPath == "" {
return errors.New("no Pulumi package found")
}
// programPath is the path to the Pulumi.yaml file. Need its parent folder.
programFolder := filepath.Dir(programPath)
archiveContents, err := archive.Process(programFolder)
if err != nil {
return errors.Wrap(err, "creating archive")
}
if printSize {
mb := float32(archiveContents.Len()) / (1024.0 * 1024.0)
fmt.Printf("Uploading %.2fMiB\n", mb)
}
parsedURL, err := url.Parse(uploadURL)
if err != nil {
return errors.Wrap(err, "parsing URL")
}
resp, err := http.DefaultClient.Do(&http.Request{
Method: "PUT",
URL: parsedURL,
ContentLength: int64(archiveContents.Len()),
Body: ioutil.NopCloser(archiveContents),
})
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.Wrap(err, "upload failed")
}
return nil
}
func (b *pulumiCloudPulumiBackend) GetLogs(stackName tokens.QName, query component.LogQuery) ([]component.LogEntry, error) {
@ -284,9 +323,7 @@ func getCloudProjectIdentifier() (*cloudProjectIdentifier, error) {
}, nil
}
// makeProgramUpdateRequest handles detecting the program, building a zip file of it, base64 encoding
// that and then returning an apitype.UpdateProgramRequest with all the relevant information to send
// to Pulumi.com
// makeProgramUpdateRequest constructs the apitype.UpdateProgramRequest based on the local machine state.
func makeProgramUpdateRequest(stackName tokens.QName) (apitype.UpdateProgramRequest, error) {
// Zip up the Pulumi program's directory, which may be a parent of CWD.
cwd, err := os.Getwd()
@ -300,18 +337,16 @@ func makeProgramUpdateRequest(stackName tokens.QName) (apitype.UpdateProgramRequ
if programPath == "" {
return apitype.UpdateProgramRequest{}, errors.Errorf("no Pulumi package found")
}
// programPath is the path to the pulumi.yaml file. Need its parent folder.
programFolder := filepath.Dir(programPath)
archive, err := archive.EncodePath(programFolder)
if err != nil {
return apitype.UpdateProgramRequest{}, errors.Errorf("creating archive: %v", err)
}
// Load the package, since we now require passing the Runtime with the update request.
pkg, err := pack.Load(programPath)
if err != nil {
return apitype.UpdateProgramRequest{}, err
}
description := ""
if pkg.Description != nil {
description = *pkg.Description
}
// Gather up configuration.
// TODO(pulumi-service/issues/221): Have pulumi.com handle the encryption/decryption.
@ -321,10 +356,10 @@ func makeProgramUpdateRequest(stackName tokens.QName) (apitype.UpdateProgramRequ
}
return apitype.UpdateProgramRequest{
Name: pkg.Name,
Runtime: pkg.Runtime,
ProgramArchive: archive,
Config: textConfig,
Name: pkg.Name,
Runtime: pkg.Runtime,
Description: description,
Config: textConfig,
}, nil
}

32
pkg/apitype/types.go
View file

@ -6,8 +6,6 @@ package apitype
import (
"fmt"
"github.com/pulumi/pulumi/pkg/tokens"
)
// User represents a Pulumi user.
@ -31,36 +29,6 @@ func (err ErrorResponse) Error() string {
return fmt.Sprintf("[%d] %s", err.Code, err.Message)
}
// UpdateProgramRequest is the request type for updating (aka deploying) a Pulumi program.
type UpdateProgramRequest struct {
// Properties from the Project file.
Name tokens.PackageName `json:"name"`
Runtime string `json:"runtime"`
// Base-64 encoded Zip archive of the program's root directory.
ProgramArchive string `json:"programArchive"`
// Configuration values.
Config map[tokens.ModuleMember]string `json:"config"`
}
// PreviewUpdateResponse is returned when previewing a potential update.
type PreviewUpdateResponse struct {
UpdateID string `json:"updateID"`
}
// UpdateProgramResponse is the response type when updating a Pulumi program.
type UpdateProgramResponse struct {
UpdateID string `json:"updateID"`
// Version is the program's new version being updated to.
Version int `json:"version"`
}
// DestroyProgramResponse is the response type when destroying a Pulumi program's resources.
type DestroyProgramResponse struct {
UpdateID string `json:"updateID"`
}
// UpdateEventKind is an enum for the type of update events.
type UpdateEventKind string

32
pkg/apitype/updates.go Normal file
View file

@ -0,0 +1,32 @@
package apitype
import "github.com/pulumi/pulumi/pkg/tokens"
// UpdateProgramRequest is the request type for updating (aka deploying) a Pulumi program.
type UpdateProgramRequest struct {
// Properties from the Project file.
Name tokens.PackageName `json:"name"`
Runtime string `json:"runtime"`
Description string `json:"description"`
// Configuration values.
Config map[tokens.ModuleMember]string `json:"config"`
}
// UpdateProgramResponse is the result of an update program request.
type UpdateProgramResponse struct {
// UpdateID is the opaque identifier of the requested update. This value is needed to begin
// an update, as well as poll for its progress.
UpdateID string `json:"updateID"`
// UploadURL is a URL the client can use to upload their program's contents into. Ignored
// for destroys.
UploadURL string `json:"uploadURL"`
}
// StartUpdateResponse is the result of the command to start an update.
type StartUpdateResponse struct {
// Version is the version of the program once the update is complete.
// (Will be the current, unchanged value for previews.)
Version int `json:"version"`
}

26
pkg/util/archive/archive.go
View file

@ -1,15 +1,13 @@
// Copyright 2016-2017, Pulumi Corporation. All rights reserved.
// Package archive provides support for creating zip archives of local folders and returning them
// as a base-64 encoded string. (Which may be rather large.) This is how we pass Pulumi program
// source to the Cloud for hosted scenarios, so the program can execute in a different environment
// and create the resources off of the local machine.
// Package archive provides support for creating zip archives of local folders and returning the
// in-memory buffer. This is how we pass Pulumi program source to the Cloud for hosted scenarios,
// for execution in a different environment and creating the resources off of the local machine.
package archive
import (
"archive/zip"
"bytes"
"encoding/base64"
"fmt"
"io"
"os"
@ -17,25 +15,19 @@ import (
"strings"
)
// EncodePath returns a base-64 encoded archive of the provided file path.
func EncodePath(path string) (string, error) {
// Process returns an in-memory buffer with the archived contents of the provided file path.
func Process(path string) (*bytes.Buffer, error) {
buffer := &bytes.Buffer{}
encoder := base64.NewEncoder(base64.StdEncoding, buffer)
writer := zip.NewWriter(encoder)
writer := zip.NewWriter(buffer)
if err := addPathToZip(writer, path, path); err != nil {
return "", err
return nil, err
}
if err := writer.Close(); err != nil {
return "", err
}
if err := encoder.Close(); err != nil {
return "", err
return nil, err
}
return buffer.String(), nil
return buffer, nil
}
// addPathToZip adds all the files in a given directory to a zip archive. All files in the archive are relative to the