[cli] Emit JSON events for updates via --json flag (#8275)
This commit is contained in:
Komal
GitHub
parent
c8e117a37d
commit
2f433d64b7
|
|
@ -1,8 +1,17 @@
|
|||
### Improvements
|
||||
|
||||
* reformat error message string in `sdk/go/common/diag/errors.go`
|
||||
- Clarify error message string in `sdk/go/common/diag/errors.go`
|
||||
[#8284](https://github.com/pulumi/pulumi/pull/8284)
|
||||
|
||||
- [cli] Add `--json` flag to `up`, `destroy` and `refresh`.
|
||||
|
||||
Passing the `--json` flag to `up`, `destroy` and `refresh` will stream JSON events from the engine to stdout.
|
||||
For `preview`, the existing functionality of outputting a JSON object at the end of preview is maintained.
|
||||
However, the streaming output can be extended to `preview` by using the `PULUMI_ENABLE_STREAMING_JSON_PREVIEW` environment variable.
|
||||
|
||||
[#8275](https://github.com/pulumi/pulumi/pull/8275)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- [sdk/dotnet] - Fix a race condition when detecting exceptions in stack creation
|
||||
[#8294](https://github.com/pulumi/pulumi/pull/8294)
|
||||
[#8294](https://github.com/pulumi/pulumi/pull/8294)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
|
||||
)
|
||||
|
|
@ -42,10 +43,14 @@ func ShowEvents(
|
|||
events, done = startEventLogger(events, done, opts)
|
||||
}
|
||||
|
||||
streamPreview := cmdutil.IsTruthy(os.Getenv("PULUMI_ENABLE_STREAMING_JSON_PREVIEW"))
|
||||
|
||||
if opts.JSONDisplay {
|
||||
// TODO[pulumi/pulumi#2390]: enable JSON display for real deployments.
|
||||
contract.Assertf(isPreview, "JSON display only available in preview mode")
|
||||
ShowJSONEvents(op, action, events, done, opts)
|
||||
if isPreview && !streamPreview {
|
||||
ShowPreviewDigest(events, done, opts)
|
||||
} else {
|
||||
ShowJSONEvents(events, done, opts)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -64,6 +69,34 @@ func ShowEvents(
|
|||
}
|
||||
}
|
||||
|
||||
func logJSONEvent(encoder *json.Encoder, event engine.Event, opts Options, seq int) error {
|
||||
apiEvent, err := ConvertEngineEvent(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
apiEvent.Sequence = seq
|
||||
apiEvent.Timestamp = int(time.Now().Unix())
|
||||
// If opts.Color == "never" (i.e. NO_COLOR is specified or --color=never), clean up the color directives
|
||||
// from the emitted events.
|
||||
if opts.Color == colors.Never {
|
||||
switch {
|
||||
case apiEvent.DiagnosticEvent != nil:
|
||||
apiEvent.DiagnosticEvent.Message = colors.Never.Colorize(apiEvent.DiagnosticEvent.Message)
|
||||
apiEvent.DiagnosticEvent.Prefix = colors.Never.Colorize(apiEvent.DiagnosticEvent.Prefix)
|
||||
apiEvent.DiagnosticEvent.Color = string(colors.Never)
|
||||
case apiEvent.StdoutEvent != nil:
|
||||
apiEvent.StdoutEvent.Message = colors.Never.Colorize(apiEvent.StdoutEvent.Message)
|
||||
apiEvent.StdoutEvent.Color = string(colors.Never)
|
||||
case apiEvent.PolicyEvent != nil:
|
||||
apiEvent.PolicyEvent.Message = colors.Never.Colorize(apiEvent.PolicyEvent.Message)
|
||||
apiEvent.PolicyEvent.Color = string(colors.Never)
|
||||
}
|
||||
}
|
||||
|
||||
return encoder.Encode(apiEvent)
|
||||
}
|
||||
|
||||
func startEventLogger(events <-chan engine.Event, done chan<- bool, opts Options) (<-chan engine.Event, chan<- bool) {
|
||||
// Before moving further, attempt to open the log file.
|
||||
logFile, err := os.Create(opts.EventLogPath)
|
||||
|
|
@ -82,38 +115,11 @@ func startEventLogger(events <-chan engine.Event, done chan<- bool, opts Options
|
|||
sequence := 0
|
||||
encoder := json.NewEncoder(logFile)
|
||||
encoder.SetEscapeHTML(false)
|
||||
logEvent := func(e engine.Event) error {
|
||||
apiEvent, err := ConvertEngineEvent(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apiEvent.Sequence, sequence = sequence, sequence+1
|
||||
apiEvent.Timestamp = int(time.Now().Unix())
|
||||
|
||||
// If opts.Color == "never" (i.e. NO_COLOR is specified or --color=never), clean up the color directives
|
||||
// from the emitted logs.
|
||||
if opts.Color == colors.Never {
|
||||
switch {
|
||||
case apiEvent.DiagnosticEvent != nil:
|
||||
apiEvent.DiagnosticEvent.Message = colors.Never.Colorize(apiEvent.DiagnosticEvent.Message)
|
||||
apiEvent.DiagnosticEvent.Prefix = colors.Never.Colorize(apiEvent.DiagnosticEvent.Prefix)
|
||||
apiEvent.DiagnosticEvent.Color = string(colors.Never)
|
||||
case apiEvent.StdoutEvent != nil:
|
||||
apiEvent.StdoutEvent.Message = colors.Never.Colorize(apiEvent.StdoutEvent.Message)
|
||||
apiEvent.StdoutEvent.Color = string(colors.Never)
|
||||
case apiEvent.PolicyEvent != nil:
|
||||
apiEvent.PolicyEvent.Message = colors.Never.Colorize(apiEvent.PolicyEvent.Message)
|
||||
apiEvent.PolicyEvent.Color = string(colors.Never)
|
||||
}
|
||||
}
|
||||
|
||||
return encoder.Encode(apiEvent)
|
||||
}
|
||||
|
||||
for e := range events {
|
||||
if err = logEvent(e); err != nil {
|
||||
if err = logJSONEvent(encoder, e, opts, sequence); err != nil {
|
||||
logging.V(7).Infof("failed to log event: %v", err)
|
||||
}
|
||||
sequence++
|
||||
|
||||
outEvents <- e
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ package display
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/pulumi/pulumi/pkg/v3/engine"
|
||||
|
|
@ -89,25 +90,46 @@ func stateForJSONOutput(s *resource.State, opts Options) *resource.State {
|
|||
s.ImportID)
|
||||
}
|
||||
|
||||
// ShowJSONEvents renders engine events from a preview into a well-formed JSON document. Note that this does not
|
||||
// ShowJSONEvents renders incremental engine events to stdout.
|
||||
func ShowJSONEvents(events <-chan engine.Event, done chan<- bool, opts Options) {
|
||||
// Ensure we close the done channel before exiting.
|
||||
defer func() { close(done) }()
|
||||
|
||||
sequence := 0
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetEscapeHTML(false)
|
||||
for e := range events {
|
||||
if err := logJSONEvent(encoder, e, opts, sequence); err != nil {
|
||||
logging.V(7).Infof("failed to log event: %v", err)
|
||||
}
|
||||
sequence++
|
||||
|
||||
// In the event of cancellation, break out of the loop.
|
||||
if e.Type == engine.CancelEvent {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ShowPreviewDigest renders engine events from a preview into a well-formed JSON document. Note that this does not
|
||||
// emit events incrementally so that it can guarantee anything emitted to stdout is well-formed. This means that,
|
||||
// if used interactively, the experience will lead to potentially very long pauses. If run in CI, it is up to the
|
||||
// end user to ensure that output is periodically printed to prevent tools from thinking preview has hung.
|
||||
func ShowJSONEvents(op string, action apitype.UpdateKind, events <-chan engine.Event, done chan<- bool, opts Options) {
|
||||
func ShowPreviewDigest(events <-chan engine.Event, done chan<- bool, opts Options) {
|
||||
// Ensure we close the done channel before exiting.
|
||||
defer func() { close(done) }()
|
||||
|
||||
// Now loop and accumulate our digest until the event stream is closed, or we hit a cancellation.
|
||||
var digest previewDigest
|
||||
for e := range events {
|
||||
// In the event of cancelation, break out of the loop immediately.
|
||||
// In the event of cancellation, break out of the loop immediately.
|
||||
if e.Type == engine.CancelEvent {
|
||||
break
|
||||
}
|
||||
|
||||
// For all other events, use the payload to build up the JSON digest we'll emit later.
|
||||
switch e.Type {
|
||||
// Events ocurring early:
|
||||
// Events occurring early:
|
||||
case engine.PreludeEvent:
|
||||
// Capture the config map from the prelude. Note that all secrets will remain blinded for safety.
|
||||
digest.Config = e.Payload().(engine.PreludeEventPayload).Config
|
||||
|
|
@ -160,7 +182,7 @@ func ShowJSONEvents(op string, action apitype.UpdateKind, events <-chan engine.E
|
|||
if err == nil {
|
||||
step.OldState = &res
|
||||
} else {
|
||||
logging.V(7).Infof("not adding old state as there was an error serialzing: %s", err)
|
||||
logging.V(7).Infof("not adding old state as there was an error serializing: %s", err)
|
||||
}
|
||||
}
|
||||
if m.New != nil {
|
||||
|
|
@ -169,18 +191,17 @@ func ShowJSONEvents(op string, action apitype.UpdateKind, events <-chan engine.E
|
|||
if err == nil {
|
||||
step.NewState = &res
|
||||
} else {
|
||||
logging.V(7).Infof("not adding new state as there was an error serialzing: %s", err)
|
||||
logging.V(7).Infof("not adding new state as there was an error serializing: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
digest.Steps = append(digest.Steps, step)
|
||||
}
|
||||
case engine.ResourceOutputsEvent, engine.ResourceOperationFailed:
|
||||
// Because we are only JSON serializing previews, we don't need to worry about outputs
|
||||
// resolving or operations failing. In the future, if we serialize actual deployments, we will
|
||||
// need to come up with a scheme for matching the failure to the associated step.
|
||||
// Because we are only JSON serializing previews, we don't need to worry about outputs
|
||||
// resolving or operations failing.
|
||||
|
||||
// Events ocurring late:
|
||||
// Events occurring late:
|
||||
case engine.PolicyViolationEvent:
|
||||
// At this point in time, we don't handle policy events in JSON serialization
|
||||
continue
|
||||
|
|
@ -194,7 +215,6 @@ func ShowJSONEvents(op string, action apitype.UpdateKind, events <-chan engine.E
|
|||
contract.Failf("unknown event type '%s'", e.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, go ahead and render the JSON to stdout.
|
||||
out, err := json.MarshalIndent(&digest, "", " ")
|
||||
contract.Assertf(err == nil, "unexpected JSON error: %v", err)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ func newDestroyCmd() *cobra.Command {
|
|||
var execAgent string
|
||||
|
||||
// Flags for engine.UpdateOptions.
|
||||
var jsonDisplay bool
|
||||
var diffDisplay bool
|
||||
var eventLogPath string
|
||||
var parallel int
|
||||
|
|
@ -91,6 +92,7 @@ func newDestroyCmd() *cobra.Command {
|
|||
Type: displayType,
|
||||
EventLogPath: eventLogPath,
|
||||
Debug: debug,
|
||||
JSONDisplay: jsonDisplay,
|
||||
}
|
||||
|
||||
// we only suppress permalinks if the user passes true. the default is an empty string
|
||||
|
|
@ -169,7 +171,7 @@ func newDestroyCmd() *cobra.Command {
|
|||
Scopes: cancellationScopes,
|
||||
})
|
||||
|
||||
if res == nil && len(*targets) == 0 {
|
||||
if res == nil && len(*targets) == 0 && !jsonDisplay {
|
||||
fmt.Printf("The resources in the stack have been deleted, but the history and configuration "+
|
||||
"associated with the stack are still maintained. \nIf you want to remove the stack "+
|
||||
"completely, run 'pulumi stack rm %s'.\n", s.Ref())
|
||||
|
|
@ -205,6 +207,9 @@ func newDestroyCmd() *cobra.Command {
|
|||
cmd.PersistentFlags().BoolVar(
|
||||
&diffDisplay, "diff", false,
|
||||
"Display operation as a rich diff showing the overall change")
|
||||
cmd.Flags().BoolVarP(
|
||||
&jsonDisplay, "json", "j", false,
|
||||
"Serialize the destroy diffs, operations, and overall output as JSON")
|
||||
cmd.PersistentFlags().IntVarP(
|
||||
¶llel, "parallel", "p", defaultParallel,
|
||||
"Allow P resource operations to run in parallel at once (1 for no parallelism). Defaults to unbounded.")
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ func newRefreshCmd() *cobra.Command {
|
|||
var stack string
|
||||
|
||||
// Flags for engine.UpdateOptions.
|
||||
var jsonDisplay bool
|
||||
var diffDisplay bool
|
||||
var eventLogPath string
|
||||
var parallel int
|
||||
|
|
@ -89,6 +90,7 @@ func newRefreshCmd() *cobra.Command {
|
|||
Type: displayType,
|
||||
EventLogPath: eventLogPath,
|
||||
Debug: debug,
|
||||
JSONDisplay: jsonDisplay,
|
||||
}
|
||||
|
||||
// we only suppress permalinks if the user passes true. the default is an empty string
|
||||
|
|
@ -198,6 +200,9 @@ func newRefreshCmd() *cobra.Command {
|
|||
cmd.PersistentFlags().BoolVar(
|
||||
&diffDisplay, "diff", false,
|
||||
"Display operation as a rich diff showing the overall change")
|
||||
cmd.Flags().BoolVarP(
|
||||
&jsonDisplay, "json", "j", false,
|
||||
"Serialize the refresh diffs, operations, and overall output as JSON")
|
||||
cmd.PersistentFlags().IntVarP(
|
||||
¶llel, "parallel", "p", defaultParallel,
|
||||
"Allow P resource operations to run in parallel at once (1 for no parallelism). Defaults to unbounded.")
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ import (
|
|||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/pulumi/pulumi/pkg/v3/backend"
|
||||
"github.com/pulumi/pulumi/pkg/v3/backend/display"
|
||||
"github.com/pulumi/pulumi/pkg/v3/engine"
|
||||
|
|
@ -34,7 +36,6 @@ import (
|
|||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/util/result"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -55,6 +56,7 @@ func newUpCmd() *cobra.Command {
|
|||
var client string
|
||||
|
||||
// Flags for engine.UpdateOptions.
|
||||
var jsonDisplay bool
|
||||
var policyPackPaths []string
|
||||
var policyPackConfigPaths []string
|
||||
var diffDisplay bool
|
||||
|
|
@ -379,6 +381,7 @@ func newUpCmd() *cobra.Command {
|
|||
Type: displayType,
|
||||
EventLogPath: eventLogPath,
|
||||
Debug: debug,
|
||||
JSONDisplay: jsonDisplay,
|
||||
}
|
||||
|
||||
// we only suppress permalinks if the user passes true. the default is an empty string
|
||||
|
|
@ -464,6 +467,9 @@ func newUpCmd() *cobra.Command {
|
|||
cmd.PersistentFlags().BoolVar(
|
||||
&diffDisplay, "diff", false,
|
||||
"Display operation as a rich diff showing the overall change")
|
||||
cmd.Flags().BoolVarP(
|
||||
&jsonDisplay, "json", "j", false,
|
||||
"Serialize the update diffs, operations, and overall output as JSON")
|
||||
cmd.PersistentFlags().IntVarP(
|
||||
¶llel, "parallel", "p", defaultParallel,
|
||||
"Allow P resource operations to run in parallel at once (1 for no parallelism). Defaults to unbounded.")
|
||||
|
|
|
|||
|
|
@ -276,6 +276,10 @@ type ProgramTestOptions struct {
|
|||
|
||||
// If set, this hook is called after the `pulumi preview` command has completed.
|
||||
PreviewCompletedHook func(dir string) error
|
||||
|
||||
// JSONOutput indicates that the `--json` flag should be passed to `up`, `preview`,
|
||||
// `refresh` and `destroy` commands.
|
||||
JSONOutput bool
|
||||
}
|
||||
|
||||
func (opts *ProgramTestOptions) GetDebugLogLevel() int {
|
||||
|
|
@ -832,7 +836,7 @@ func (pt *ProgramTester) runPulumiCommand(name string, args []string, wd string,
|
|||
return false, nil, nil
|
||||
}
|
||||
|
||||
// someother error, fail
|
||||
// some other error, fail
|
||||
return false, nil, runerr
|
||||
},
|
||||
})
|
||||
|
|
@ -1152,6 +1156,9 @@ func (pt *ProgramTester) TestLifeCycleDestroy() error {
|
|||
if pt.opts.GetDebugUpdates() {
|
||||
destroy = append(destroy, "-d")
|
||||
}
|
||||
if pt.opts.JSONOutput {
|
||||
destroy = append(destroy, "--json")
|
||||
}
|
||||
if err := pt.runPulumiCommand("pulumi-destroy", destroy, pt.projdir, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1213,6 +1220,9 @@ func (pt *ProgramTester) TestPreviewUpdateAndEdits() error {
|
|||
if pt.opts.GetDebugUpdates() {
|
||||
refresh = append(refresh, "-d")
|
||||
}
|
||||
if pt.opts.JSONOutput {
|
||||
refresh = append(refresh, "--json")
|
||||
}
|
||||
if !pt.opts.ExpectRefreshChanges {
|
||||
refresh = append(refresh, "--expect-no-changes")
|
||||
}
|
||||
|
|
@ -1250,6 +1260,10 @@ func (pt *ProgramTester) PreviewAndUpdate(dir string, name string, shouldFail, e
|
|||
preview = append(preview, "-d")
|
||||
update = append(update, "-d")
|
||||
}
|
||||
if pt.opts.JSONOutput {
|
||||
preview = append(preview, "--json")
|
||||
update = append(update, "--json")
|
||||
}
|
||||
if expectNopPreview {
|
||||
preview = append(preview, "--expect-no-changes")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,18 +4,21 @@ package ints
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
|
||||
|
|
@ -828,3 +831,64 @@ func TestRotatePassphrase(t *testing.T) {
|
|||
e.Stdin, e.Passphrase = nil, "qwerty"
|
||||
e.RunCommand("pulumi", "config", "get", "foo")
|
||||
}
|
||||
|
||||
var previewSummaryRegex = regexp.MustCompile(
|
||||
`{\s+"steps": \[[\s\S]+],\s+"duration": \d+,\s+"changeSummary": {[\s\S]+}\s+}`)
|
||||
|
||||
func assertOutputContainsEvent(t *testing.T, evt apitype.EngineEvent, output string) {
|
||||
evtJSON := bytes.Buffer{}
|
||||
encoder := json.NewEncoder(&evtJSON)
|
||||
encoder.SetEscapeHTML(false)
|
||||
err := encoder.Encode(evt)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, output, evtJSON.String())
|
||||
}
|
||||
|
||||
func TestJSONOutput(t *testing.T) {
|
||||
stdout := &bytes.Buffer{}
|
||||
|
||||
// Test without env var for streaming preview (should print previewSummary).
|
||||
integration.ProgramTest(t, &integration.ProgramTestOptions{
|
||||
Dir: filepath.Join("stack_outputs", "nodejs"),
|
||||
Dependencies: []string{"@pulumi/pulumi"},
|
||||
Stdout: stdout,
|
||||
Verbose: true,
|
||||
JSONOutput: true,
|
||||
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
|
||||
output := stdout.String()
|
||||
|
||||
// Check that the previewSummary is present.
|
||||
assert.Regexp(t, previewSummaryRegex, output)
|
||||
|
||||
// Check that each event present in the event stream is also in stdout.
|
||||
for _, evt := range stack.Events {
|
||||
assertOutputContainsEvent(t, evt, output)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestJSONOutputWithStreamingPreview(t *testing.T) {
|
||||
stdout := &bytes.Buffer{}
|
||||
|
||||
// Test with env var for streaming preview (should *not* print previewSummary).
|
||||
integration.ProgramTest(t, &integration.ProgramTestOptions{
|
||||
Dir: filepath.Join("stack_outputs", "nodejs"),
|
||||
Dependencies: []string{"@pulumi/pulumi"},
|
||||
Stdout: stdout,
|
||||
Verbose: true,
|
||||
JSONOutput: true,
|
||||
Env: []string{"PULUMI_ENABLE_STREAMING_JSON_PREVIEW=1"},
|
||||
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
|
||||
output := stdout.String()
|
||||
|
||||
// Check that the previewSummary is *not* present.
|
||||
assert.NotRegexp(t, previewSummaryRegex, output)
|
||||
|
||||
// Check that each event present in the event stream is also in stdout.
|
||||
for _, evt := range stack.Events {
|
||||
assertOutputContainsEvent(t, evt, output)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue