maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add pagination to pulumi stack history (#6292)

Browse source 
replaces the unreleased `--limit` flag with `--page` and `--page-size` to support full pagination
This commit is contained in:
Evan Boyle 2021-02-09 16:20:01 -08:00 committed by GitHub
parent e2b48d2f20
commit 8e58f5d682
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 121 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -3,6 +3,9 @@ CHANGELOG
## HEAD (Unreleased)
- [CLI] Add pagination options to `pulumi stack history` (`--page`, `--page-size`). These replace the `--limit` flag.
[#6292](https://github.com/pulumi/pulumi/pull/6292)
- [CLI, automation/*] Add `--limit` flag to `pulumi stack history` command and consume this from Automation API SDKs to improve performance of stack updates.
[#6257](https://github.com/pulumi/pulumi/pull/6257)

2
pkg/backend/backend.go
View file

@ -171,7 +171,7 @@ type Backend interface {
// GetHistory returns all updates for the stack. The returned UpdateInfo slice will be in
// descending order (newest first).
GetHistory(ctx context.Context, stackRef StackReference, limit int) ([]UpdateInfo, error)
GetHistory(ctx context.Context, stackRef StackReference, pageSize int, page int) ([]UpdateInfo, error)
// GetLogs fetches a list of log entries for the given stack, with optional filtering/querying.
GetLogs(ctx context.Context, stack Stack, cfg StackConfiguration,
query operations.LogQuery) ([]operations.LogEntry, error)

7
pkg/backend/filestate/backend.go
View file

@ -399,7 +399,7 @@ func (b *localBackend) RenameStack(ctx context.Context, stack backend.Stack,
func (b *localBackend) GetLatestConfiguration(ctx context.Context,
stack backend.Stack) (config.Map, error) {
hist, err := b.GetHistory(ctx, stack.Ref(), 1 /*limit*/)
hist, err := b.GetHistory(ctx, stack.Ref(), 1 /*pageSize*/, 1 /*page*/)
if err != nil {
return nil, err
}
@ -626,9 +626,10 @@ func (b *localBackend) query(ctx context.Context, op backend.QueryOperation,
func (b *localBackend) GetHistory(
ctx context.Context,
stackRef backend.StackReference,
limit int) ([]backend.UpdateInfo, error) {
pageSize int,
page int) ([]backend.UpdateInfo, error) {
stackName := stackRef.Name()
updates, err := b.getHistory(stackName, limit)
updates, err := b.getHistory(stackName, pageSize, page)
if err != nil {
return nil, err
}

46
pkg/backend/filestate/state.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pulumi/pulumi/pkg/v2/engine"
"github.com/pkg/errors"
"gocloud.dev/blob"
"gocloud.dev/gcerrors"
"github.com/pulumi/pulumi/pkg/v2/backend"
@ -313,13 +314,12 @@ func (b *localBackend) backupDirectory(stack tokens.QName) string {
// getHistory returns locally stored update history. The first element of the result will be
// the most recent update record.
func (b *localBackend) getHistory(name tokens.QName, limit int) ([]backend.UpdateInfo, error) {
func (b *localBackend) getHistory(name tokens.QName, pageSize int, page int) ([]backend.UpdateInfo, error) {
contract.Require(name != "", "name")
dir := b.historyDirectory(name)
// TODO: we could consider optimizing the list operation using `limit`.
// Unfortunately, this is mildly invasive given the gocloud List API and
// the fact that are results are returned in reverse order.
// TODO: we could consider optimizing the list operation using `page` and `pageSize`.
// Unfortunately, this is mildly invasive given the gocloud List API.
allFiles, err := listBucket(b.bucket, dir)
if err != nil {
// History doesn't exist until a stack has been updated.
@ -329,26 +329,42 @@ func (b *localBackend) getHistory(name tokens.QName, limit int) ([]backend.Updat
return nil, err
}
var updates []backend.UpdateInfo
if limit > 0 {
if limit > len(allFiles) {
limit = len(allFiles)
}
limit = len(allFiles) - limit
}
var historyEntries []*blob.ListObject
// filter down to just history entries, reversing list to be in most recent order.
// listBucket returns the array sorted by file name, but because of how we name files, older updates come before
// newer ones. Loop backwards so we added the newest updates to the array we will return first.
for i := len(allFiles) - 1; i >= limit; i-- {
// newer ones.
for i := len(allFiles) - 1; i >= 0; i-- {
file := allFiles[i]
filepath := file.Key
// Open all of the history files, ignoring the checkpoints.
// ignore checkpoints
if !strings.HasSuffix(filepath, ".history.json") {
continue
}
historyEntries = append(historyEntries, file)
}
start := 0
end := len(historyEntries) - 1
if pageSize > 0 {
if page < 1 {
page = 1
}
start = (page - 1) * pageSize
end = start + pageSize - 1
if end > len(historyEntries)-1 {
end = len(historyEntries) - 1
}
}
var updates []backend.UpdateInfo
for i := start; i <= end; i++ {
file := historyEntries[i]
filepath := file.Key
var update backend.UpdateInfo
b, err := b.bucket.ReadAll(context.TODO(), filepath)
if err != nil {

6
pkg/backend/httpstate/backend.go
View file

@ -1100,13 +1100,15 @@ func (b *cloudBackend) CancelCurrentUpdate(ctx context.Context, stackRef backend
func (b *cloudBackend) GetHistory(
ctx context.Context,
stackRef backend.StackReference, limit int) ([]backend.UpdateInfo, error) {
stackRef backend.StackReference,
pageSize int,
page int) ([]backend.UpdateInfo, error) {
stack, err := b.getCloudStackIdentifier(stackRef)
if err != nil {
return nil, err
}
updates, err := b.client.GetStackUpdates(ctx, stack, limit)
updates, err := b.client.GetStackUpdates(ctx, stack, pageSize, page)
if err != nil {
return nil, err
}

8
pkg/backend/httpstate/client/client.go
View file

@ -376,11 +376,15 @@ func (pc *Client) DecryptValue(ctx context.Context, stack StackIdentifier, ciphe
func (pc *Client) GetStackUpdates(
ctx context.Context,
stack StackIdentifier,
pageSize int) ([]apitype.UpdateInfo, error) {
pageSize int,
page int) ([]apitype.UpdateInfo, error) {
var response apitype.GetHistoryResponse
path := getStackPath(stack, "updates")
if pageSize > 0 {
path += fmt.Sprintf("?pageSize=%d&page=1", pageSize)
if page < 1 {
page = 1
}
path += fmt.Sprintf("?pageSize=%d&page=%d", pageSize, page)
}
if err := pc.restCall(ctx, "GET", path, nil, nil, &response); err != nil {
return nil, err

9
pkg/backend/mock.go
View file

@ -47,7 +47,7 @@ type MockBackend struct {
GetStackCrypterF func(StackReference) (config.Crypter, error)
QueryF func(context.Context, QueryOperation) result.Result
GetLatestConfigurationF func(context.Context, Stack) (config.Map, error)
GetHistoryF func(context.Context, StackReference, int) ([]UpdateInfo, error)
GetHistoryF func(context.Context, StackReference, int, int) ([]UpdateInfo, error)
GetStackTagsF func(context.Context, Stack) (map[apitype.StackTagName]string, error)
UpdateStackTagsF func(context.Context, Stack, map[apitype.StackTagName]string) error
ExportDeploymentF func(context.Context, Stack) (*apitype.UntypedDeployment, error)
@ -237,9 +237,12 @@ func (be *MockBackend) Query(ctx context.Context, op QueryOperation) result.Resu
panic("not implemented")
}
func (be *MockBackend) GetHistory(ctx context.Context, stackRef StackReference, limit int) ([]UpdateInfo, error) {
func (be *MockBackend) GetHistory(ctx context.Context,
stackRef StackReference,
pageSize int,
page int) ([]UpdateInfo, error) {
if be.GetHistoryF != nil {
return be.GetHistoryF(ctx, stackRef, limit)
return be.GetHistoryF(ctx, stackRef, pageSize, page)
}
panic("not implemented")
}

13
pkg/cmd/pulumi/history.go
View file

@ -28,7 +28,8 @@ func newHistoryCmd() *cobra.Command {
var stack string
var jsonOut bool
var showSecrets bool
var limit int
var pageSize int
var page int
var cmd = &cobra.Command{
Use: "history",
Aliases: []string{"hist"},
@ -50,7 +51,7 @@ func newHistoryCmd() *cobra.Command {
}
b := s.Backend()
updates, err := b.GetHistory(commandContext(), s.Ref(), limit)
updates, err := b.GetHistory(commandContext(), s.Ref(), pageSize, page)
if err != nil {
return errors.Wrap(err, "getting history")
}
@ -67,7 +68,7 @@ func newHistoryCmd() *cobra.Command {
return displayUpdatesJSON(updates, decrypter)
}
return displayUpdatesConsole(updates, opts)
return displayUpdatesConsole(updates, page, opts)
}),
}
cmd.PersistentFlags().StringVarP(
@ -78,7 +79,9 @@ func newHistoryCmd() *cobra.Command {
"Show secret values when listing config instead of displaying blinded values")
cmd.PersistentFlags().BoolVarP(
&jsonOut, "json", "j", false, "Emit output as JSON")
cmd.PersistentFlags().IntVarP(
&limit, "limit", "l", 0, "Limit the number of entries returned, defaults to all")
cmd.PersistentFlags().IntVar(
&pageSize, "page-size", 0, "Used with 'page' to control number of results returned")
cmd.PersistentFlags().IntVar(
&page, "page", 0, "Used with 'page-size' to paginate results")
return cmd
}

19
pkg/cmd/pulumi/stack_history.go
View file

@ -24,7 +24,8 @@ func newStackHistoryCmd() *cobra.Command {
var stack string
var jsonOut bool
var showSecrets bool
var limit int
var pageSize int
var page int
cmd := &cobra.Command{
Use: "history",
@ -43,7 +44,7 @@ This command displays data about previous updates for a stack.`,
return err
}
b := s.Backend()
updates, err := b.GetHistory(commandContext(), s.Ref(), limit)
updates, err := b.GetHistory(commandContext(), s.Ref(), pageSize, page)
if err != nil {
return errors.Wrap(err, "getting history")
}
@ -60,7 +61,7 @@ This command displays data about previous updates for a stack.`,
return displayUpdatesJSON(updates, decrypter)
}
return displayUpdatesConsole(updates, opts)
return displayUpdatesConsole(updates, page, opts)
}),
}
@ -72,8 +73,10 @@ This command displays data about previous updates for a stack.`,
"Show secret values when listing config instead of displaying blinded values")
cmd.PersistentFlags().BoolVarP(
&jsonOut, "json", "j", false, "Emit output as JSON")
cmd.PersistentFlags().IntVarP(
&limit, "limit", "l", 0, "Limit the number of entries returned, defaults to all")
cmd.PersistentFlags().IntVar(
&pageSize, "page-size", 0, "Used with 'page' to control number of results returned")
cmd.PersistentFlags().IntVar(
&page, "page", 0, "Used with 'page-size' to paginate results")
return cmd
}
@ -148,8 +151,12 @@ func displayUpdatesJSON(updates []backend.UpdateInfo, decrypter config.Decrypter
return printJSON(updatesJSON)
}
func displayUpdatesConsole(updates []backend.UpdateInfo, opts display.Options) error {
func displayUpdatesConsole(updates []backend.UpdateInfo, page int, opts display.Options) error {
if len(updates) == 0 {
if page > 1 {
fmt.Printf("No stack updates found on page '%d'\n", page)
return nil
}
fmt.Println("Stack has never been updated")
return nil
}

5
sdk/go/x/auto/example_test.go
View file

@ -1034,8 +1034,9 @@ func ExampleStack_History() {
ctx := context.Background()
stackName := FullyQualifiedStackName("org", "project", "stack")
stack, _ := SelectStackLocalSource(ctx, stackName, filepath.Join(".", "program"))
limit := 0 // fetch all history entries
hist, _ := stack.History(ctx, limit)
pageSize := 0 // fetch all history entries, don't paginate
page := 0 // fetch all history entries, don't paginate
hist, _ := stack.History(ctx, pageSize, page)
// last operation start time
fmt.Println(hist[0].StartTime)
}

16
sdk/go/x/auto/stack.go
View file

@ -324,7 +324,7 @@ func (s *Stack) Up(ctx context.Context, opts ...optup.Option) (UpResult, error)
return res, err
}
history, err := s.History(ctx, 1 /*limit*/)
history, err := s.History(ctx, 1 /*pageSize*/, 1 /*page*/)
if err != nil {
return res, err
}
@ -384,7 +384,7 @@ func (s *Stack) Refresh(ctx context.Context, opts ...optrefresh.Option) (Refresh
return res, newAutoError(errors.Wrap(err, "failed to refresh stack"), stdout, stderr, code)
}
history, err := s.History(ctx, 1 /*limit*/)
history, err := s.History(ctx, 1 /*pageSize*/, 1 /*page*/)
if err != nil {
return res, errors.Wrap(err, "failed to refresh stack")
}
@ -444,7 +444,7 @@ func (s *Stack) Destroy(ctx context.Context, opts ...optdestroy.Option) (Destroy
return res, newAutoError(errors.Wrap(err, "failed to destroy stack"), stdout, stderr, code)
}
history, err := s.History(ctx, 1 /*limit*/)
history, err := s.History(ctx, 1 /*pageSize*/, 1 /*page*/)
if err != nil {
return res, errors.Wrap(err, "failed to destroy stack")
}
@ -511,14 +511,18 @@ func (s *Stack) Outputs(ctx context.Context) (OutputMap, error) {
// History returns a list summarizing all previous and current results from Stack lifecycle operations
// (up/preview/refresh/destroy).
func (s *Stack) History(ctx context.Context, limit int) ([]UpdateSummary, error) {
func (s *Stack) History(ctx context.Context, pageSize int, page int) ([]UpdateSummary, error) {
err := s.Workspace().SelectStack(ctx, s.Name())
if err != nil {
return nil, errors.Wrap(err, "failed to get stack history")
}
args := []string{"history", "--json", "--show-secrets"}
if limit > 0 {
args = append(args, "--limit", fmt.Sprintf("%d", limit))
if pageSize > 0 {
// default page=1 if unset when pageSize is set
if page < 1 {
page = 1
}
args = append(args, "--page-size", fmt.Sprintf("%d", pageSize), "--page", fmt.Sprintf("%d", page))
}
stdout, stderr, errCode, err := s.runPulumiCmdSync(ctx, nil /* additionalOutputs */, args...)

11
sdk/nodejs/x/automation/stack.ts
View file

@ -415,10 +415,13 @@ export class Stack {
* Returns a list summarizing all previous and current results from Stack lifecycle operations
* (up/preview/refresh/destroy).
*/
async history(limit?: number): Promise<UpdateSummary[]> {
async history(pageSize?: number, page?: number): Promise<UpdateSummary[]> {
const args = ["history", "--json", "--show-secrets"];
if (limit) {
args.push("--limit", Math.floor(limit).toString())
if (pageSize) {
if (!page || page < 1) {
page = 1
}
args.push("--page-size", Math.floor(pageSize).toString(), "--page", Math.floor(page).toString())
}
const result = await this.runPulumiCmd(args);
@ -430,7 +433,7 @@ export class Stack {
});
}
async info(): Promise<UpdateSummary | undefined> {
const history = await this.history(1 /*limit*/);
const history = await this.history(1 /*pageSize*/);
if (!history || history.length === 0) {
return undefined;
}

15
sdk/python/lib/pulumi/x/automation/stack.py
View file

@ -465,18 +465,23 @@ class Stack:
return outputs
def history(self,
limit: Optional[int] = None) -> List[UpdateSummary]:
page_size: Optional[int] = None,
page: Optional[int] = None) -> List[UpdateSummary]:
"""
Returns a list summarizing all previous and current results from Stack lifecycle operations
(up/preview/refresh/destroy).
:param limit: Limit the number of history entires to retrieve, defaults to all.
:param page_size: Paginate history entries (used in combination with page), defaults to all.
:param page: Paginate history entries (used in combination with page_size), defaults to all.
:returns: List[UpdateSummary]
"""
args = ["history", "--json", "--show-secrets"]
if limit is not None:
args.extend(["--limit", str(limit)])
if page_size is not None:
# default page=1 when page_size is set
if page is None:
page = 1
args.extend(["--page-size", str(page_size), "--page", str(page)])
result = self._run_pulumi_cmd_sync(args)
summary_list = json.loads(result.stdout)
@ -501,7 +506,7 @@ class Stack:
:returns: Optional[UpdateSummary]
"""
history = self.history(limit=1)
history = self.history(page_size=1)
if not len(history):
return None
return history[0]

15
tests/history_test.go
View file

@ -37,6 +37,15 @@ func assertHasNoHistory(e *ptesting.Environment) {
assert.Equal(e.T, "", err)
assert.Equal(e.T, "Stack has never been updated\n", out)
}
// assertNoResultsOnPage runs `pulumi history` and confirms an error that the stack has no
// updates in the given pagination window
func assertNoResultsOnPage(e *ptesting.Environment) {
// NOTE: pulumi returns with exit code 0 in this scenario.
out, err := e.RunCommand("pulumi", "history", "--page-size", "1", "--page", "10000000")
assert.Equal(e.T, "", err)
assert.Equal(e.T, "No stack updates found on page '10000000'\n", out)
}
func TestHistoryCommand(t *testing.T) {
// We fail if no stack is selected.
t.Run("NoStackSelected", func(t *testing.T) {
@ -77,6 +86,12 @@ func TestHistoryCommand(t *testing.T) {
out, err := e.RunCommand("pulumi", "history")
assert.Equal(t, "", err)
assert.Contains(t, out, "this is an updated stack")
// Confirm we see the update message in thie history output, with pagination.
out, err = e.RunCommand("pulumi", "history", "--page-size", "1")
assert.Equal(t, "", err)
assert.Contains(t, out, "this is an updated stack")
// Get an error message when we page too far
assertNoResultsOnPage(e)
// Change stack and confirm the history command honors the selected stack.
e.RunCommand("pulumi", "stack", "select", "stack-without-updates")
assertHasNoHistory(e)