maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add --limit flag to pulumi stack history and consume from automation api (#6257)

Browse source 
Adds a `--limit` flag to `pulumi stack history. This allows limiting to the last few entries rather than fetching the entirety of a stack's update history (which can be quite slow for stacks with lots of updates). Example: `pulumi stack history --limit 1` fetches the last history entry only. 

`stack.up` and related operations in the Automation API have been updated to consume this change, drastically reducing overhead.
This commit is contained in:
Evan Boyle 2021-02-08 10:49:57 -08:00 committed by GitHub
parent a045a613f2
commit eefc104c2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 70 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -3,6 +3,9 @@ CHANGELOG
## HEAD (Unreleased)
- [CLI, automation/*] Add `--limit` flag to `pulumi stack history` command and consume this from Automation API SDKs to improve performance of stack updates.
[#6257](https://github.com/pulumi/pulumi/pull/6257)
- [sdk/python] Gracefully handle monitor shutdown in the python runtime without exiting the process.
[#6249](https://github.com/pulumi/pulumi/pull/6249)

2
pkg/backend/backend.go
View file

@ -171,7 +171,7 @@ type Backend interface {
// GetHistory returns all updates for the stack. The returned UpdateInfo slice will be in
// descending order (newest first).
GetHistory(ctx context.Context, stackRef StackReference) ([]UpdateInfo, error)
GetHistory(ctx context.Context, stackRef StackReference, limit int) ([]UpdateInfo, error)
// GetLogs fetches a list of log entries for the given stack, with optional filtering/querying.
GetLogs(ctx context.Context, stack Stack, cfg StackConfiguration,
query operations.LogQuery) ([]operations.LogEntry, error)

6
pkg/backend/filestate/backend.go
View file

@ -399,7 +399,7 @@ func (b *localBackend) RenameStack(ctx context.Context, stack backend.Stack,
func (b *localBackend) GetLatestConfiguration(ctx context.Context,
stack backend.Stack) (config.Map, error) {
hist, err := b.GetHistory(ctx, stack.Ref())
hist, err := b.GetHistory(ctx, stack.Ref(), 1 /*limit*/)
if err != nil {
return nil, err
}
@ -623,9 +623,9 @@ func (b *localBackend) query(ctx context.Context, op backend.QueryOperation,
return backend.RunQuery(ctx, b, op, callerEventsOpt, b.newQuery)
}
func (b *localBackend) GetHistory(ctx context.Context, stackRef backend.StackReference) ([]backend.UpdateInfo, error) {
func (b *localBackend) GetHistory(ctx context.Context, stackRef backend.StackReference, limit int) ([]backend.UpdateInfo, error) {
stackName := stackRef.Name()
updates, err := b.getHistory(stackName)
updates, err := b.getHistory(stackName, limit)
if err != nil {
return nil, err
}

17
pkg/backend/filestate/state.go
View file

@ -18,13 +18,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/pulumi/pulumi/sdk/v2/go/common/util/retry"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/pulumi/pulumi/sdk/v2/go/common/util/retry"
"github.com/pulumi/pulumi/pkg/v2/engine"
"github.com/pkg/errors"
@ -312,10 +313,13 @@ func (b *localBackend) backupDirectory(stack tokens.QName) string {
// getHistory returns locally stored update history. The first element of the result will be
// the most recent update record.
func (b *localBackend) getHistory(name tokens.QName) ([]backend.UpdateInfo, error) {
func (b *localBackend) getHistory(name tokens.QName, limit int) ([]backend.UpdateInfo, error) {
contract.Require(name != "", "name")
dir := b.historyDirectory(name)
// TODO: we could consider optimizing the list operation using `limit`.
// Unfortunately, this is mildly invasive given the gocloud List API and
// the fact that are results are returned in reverse order.
allFiles, err := listBucket(b.bucket, dir)
if err != nil {
// History doesn't exist until a stack has been updated.
@ -327,9 +331,16 @@ func (b *localBackend) getHistory(name tokens.QName) ([]backend.UpdateInfo, erro
var updates []backend.UpdateInfo
if limit > 0 {
if limit > len(allFiles) {
limit = len(allFiles)
}
limit = len(allFiles) - limit
}
// listBucket returns the array sorted by file name, but because of how we name files, older updates come before
// newer ones. Loop backwards so we added the newest updates to the array we will return first.
for i := len(allFiles) - 1; i >= 0; i-- {
for i := len(allFiles) - 1; i >= limit; i-- {
file := allFiles[i]
filepath := file.Key

4
pkg/backend/httpstate/backend.go
View file

@ -1098,13 +1098,13 @@ func (b *cloudBackend) CancelCurrentUpdate(ctx context.Context, stackRef backend
return b.client.CancelUpdate(ctx, updateID)
}
func (b *cloudBackend) GetHistory(ctx context.Context, stackRef backend.StackReference) ([]backend.UpdateInfo, error) {
func (b *cloudBackend) GetHistory(ctx context.Context, stackRef backend.StackReference, limit int) ([]backend.UpdateInfo, error) {
stack, err := b.getCloudStackIdentifier(stackRef)
if err != nil {
return nil, err
}
updates, err := b.client.GetStackUpdates(ctx, stack)
updates, err := b.client.GetStackUpdates(ctx, stack, limit)
if err != nil {
return nil, err
}

8
pkg/backend/httpstate/client/client.go
View file

@ -373,9 +373,13 @@ func (pc *Client) DecryptValue(ctx context.Context, stack StackIdentifier, ciphe
}
// GetStackUpdates returns all updates to the indicated stack.
func (pc *Client) GetStackUpdates(ctx context.Context, stack StackIdentifier) ([]apitype.UpdateInfo, error) {
func (pc *Client) GetStackUpdates(ctx context.Context, stack StackIdentifier, pageSize int) ([]apitype.UpdateInfo, error) {
var response apitype.GetHistoryResponse
if err := pc.restCall(ctx, "GET", getStackPath(stack, "updates"), nil, nil, &response); err != nil {
path := getStackPath(stack, "updates")
if pageSize > 0 {
path += fmt.Sprintf("?pageSize=%d&page=1", pageSize)
}
if err := pc.restCall(ctx, "GET", path, nil, nil, &response); err != nil {
return nil, err
}

6
pkg/backend/mock.go
View file

@ -47,7 +47,7 @@ type MockBackend struct {
GetStackCrypterF func(StackReference) (config.Crypter, error)
QueryF func(context.Context, QueryOperation) result.Result
GetLatestConfigurationF func(context.Context, Stack) (config.Map, error)
GetHistoryF func(context.Context, StackReference) ([]UpdateInfo, error)
GetHistoryF func(context.Context, StackReference, int) ([]UpdateInfo, error)
GetStackTagsF func(context.Context, Stack) (map[apitype.StackTagName]string, error)
UpdateStackTagsF func(context.Context, Stack, map[apitype.StackTagName]string) error
ExportDeploymentF func(context.Context, Stack) (*apitype.UntypedDeployment, error)
@ -237,9 +237,9 @@ func (be *MockBackend) Query(ctx context.Context, op QueryOperation) result.Resu
panic("not implemented")
}
func (be *MockBackend) GetHistory(ctx context.Context, stackRef StackReference) ([]UpdateInfo, error) {
func (be *MockBackend) GetHistory(ctx context.Context, stackRef StackReference, limit int) ([]UpdateInfo, error) {
if be.GetHistoryF != nil {
return be.GetHistoryF(ctx, stackRef)
return be.GetHistoryF(ctx, stackRef, limit)
}
panic("not implemented")
}

6
pkg/cmd/pulumi/history.go
View file

@ -28,6 +28,7 @@ func newHistoryCmd() *cobra.Command {
var stack string
var jsonOut bool
var showSecrets bool
var limit int
var cmd = &cobra.Command{
Use: "history",
Aliases: []string{"hist"},
@ -47,8 +48,9 @@ func newHistoryCmd() *cobra.Command {
if err != nil {
return err
}
b := s.Backend()
updates, err := b.GetHistory(commandContext(), s.Ref())
updates, err := b.GetHistory(commandContext(), s.Ref(), limit)
if err != nil {
return errors.Wrap(err, "getting history")
}
@ -76,5 +78,7 @@ func newHistoryCmd() *cobra.Command {
"Show secret values when listing config instead of displaying blinded values")
cmd.PersistentFlags().BoolVarP(
&jsonOut, "json", "j", false, "Emit output as JSON")
cmd.PersistentFlags().IntVarP(
&limit, "limit", "l", 0, "Limit the number of entries returned, defaults to all")
return cmd
}

5
pkg/cmd/pulumi/stack_history.go
View file

@ -24,6 +24,7 @@ func newStackHistoryCmd() *cobra.Command {
var stack string
var jsonOut bool
var showSecrets bool
var limit int
cmd := &cobra.Command{
Use: "history",
@ -42,7 +43,7 @@ This command displays data about previous updates for a stack.`,
return err
}
b := s.Backend()
updates, err := b.GetHistory(commandContext(), s.Ref())
updates, err := b.GetHistory(commandContext(), s.Ref(), limit)
if err != nil {
return errors.Wrap(err, "getting history")
}
@ -71,6 +72,8 @@ This command displays data about previous updates for a stack.`,
"Show secret values when listing config instead of displaying blinded values")
cmd.PersistentFlags().BoolVarP(
&jsonOut, "json", "j", false, "Emit output as JSON")
cmd.PersistentFlags().IntVarP(
&limit, "limit", "l", 0, "Limit the number of entries returned, defaults to all")
return cmd
}

3
sdk/go/x/auto/example_test.go
View file

@ -1034,7 +1034,8 @@ func ExampleStack_History() {
ctx := context.Background()
stackName := FullyQualifiedStackName("org", "project", "stack")
stack, _ := SelectStackLocalSource(ctx, stackName, filepath.Join(".", "program"))
hist, _ := stack.History(ctx)
limit := 0 // fetch all history entries
hist, _ := stack.History(ctx, limit)
// last operation start time
fmt.Println(hist[0].StartTime)
}

15
sdk/go/x/auto/stack.go
View file

@ -91,13 +91,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/pulumi/pulumi/sdk/v2/go/x/auto/debug"
"io"
"regexp"
"runtime"
"strings"
"sync"
"github.com/pulumi/pulumi/sdk/v2/go/x/auto/debug"
pbempty "github.com/golang/protobuf/ptypes/empty"
"github.com/pkg/errors"
"google.golang.org/grpc"
@ -323,7 +324,7 @@ func (s *Stack) Up(ctx context.Context, opts ...optup.Option) (UpResult, error)
return res, err
}
history, err := s.History(ctx)
history, err := s.History(ctx, 1 /*limit*/)
if err != nil {
return res, err
}
@ -383,7 +384,7 @@ func (s *Stack) Refresh(ctx context.Context, opts ...optrefresh.Option) (Refresh
return res, newAutoError(errors.Wrap(err, "failed to refresh stack"), stdout, stderr, code)
}
history, err := s.History(ctx)
history, err := s.History(ctx, 1 /*limit*/)
if err != nil {
return res, errors.Wrap(err, "failed to refresh stack")
}
@ -443,7 +444,7 @@ func (s *Stack) Destroy(ctx context.Context, opts ...optdestroy.Option) (Destroy
return res, newAutoError(errors.Wrap(err, "failed to destroy stack"), stdout, stderr, code)
}
history, err := s.History(ctx)
history, err := s.History(ctx, 1 /*limit*/)
if err != nil {
return res, errors.Wrap(err, "failed to destroy stack")
}
@ -510,11 +511,15 @@ func (s *Stack) Outputs(ctx context.Context) (OutputMap, error) {
// History returns a list summarizing all previous and current results from Stack lifecycle operations
// (up/preview/refresh/destroy).
func (s *Stack) History(ctx context.Context) ([]UpdateSummary, error) {
func (s *Stack) History(ctx context.Context, limit int) ([]UpdateSummary, error) {
err := s.Workspace().SelectStack(ctx, s.Name())
if err != nil {
return nil, errors.Wrap(err, "failed to get stack history")
}
args := []string{"history", "--json", "--show-secrets"}
if limit > 0 {
args = append(args, "--limit", fmt.Sprintf("%d", limit))
}
stdout, stderr, errCode, err := s.runPulumiCmdSync(ctx, nil, /* additionalOutputs */
"history", "--json", "--show-secrets",

11
sdk/nodejs/x/automation/stack.ts
View file

@ -415,8 +415,13 @@ export class Stack {
* Returns a list summarizing all previous and current results from Stack lifecycle operations
* (up/preview/refresh/destroy).
*/
async history(): Promise<UpdateSummary[]> {
const result = await this.runPulumiCmd(["history", "--json", "--show-secrets"]);
async history(limit?: number): Promise<UpdateSummary[]> {
const args = ["history", "--json", "--show-secrets"];
if (limit) {
args.push("--limit", Math.floor(limit).toString())
}
const result = await this.runPulumiCmd(args);
return JSON.parse(result.stdout, (key, value) => {
if (key === "startTime" || key === "endTime") {
return new Date(value);
@ -425,7 +430,7 @@ export class Stack {
});
}
async info(): Promise<UpdateSummary | undefined> {
const history = await this.history();
const history = await this.history(1 /*limit*/);
if (!history || history.length === 0) {
return undefined;
}

12
sdk/python/lib/pulumi/x/automation/stack.py
View file

@ -464,14 +464,20 @@ class Stack:
outputs[key] = OutputValue(value=plaintext_outputs[key], secret=secret)
return outputs
def history(self) -> List[UpdateSummary]:
def history(self,
limit: Optional[int] = None) -> List[UpdateSummary]:
"""
Returns a list summarizing all previous and current results from Stack lifecycle operations
(up/preview/refresh/destroy).
:param limit: Limit the number of history entires to retrieve, defaults to all.
:returns: List[UpdateSummary]
"""
result = self._run_pulumi_cmd_sync(["history", "--json", "--show-secrets"])
args = ["history", "--json", "--show-secrets"]
if limit is not None:
args.extend(["--limit", str(limit)])
result = self._run_pulumi_cmd_sync(args)
summary_list = json.loads(result.stdout)
summaries: List[UpdateSummary] = []
@ -495,7 +501,7 @@ class Stack:
:returns: Optional[UpdateSummary]
"""
history = self.history()
history = self.history(limit=1)
if not len(history):
return None
return history[0]