pulumi/cmd/stack_graph.go

253 lines
7.5 KiB
Go
Raw Normal View History

2018-05-22 21:43:36 +02:00
// Copyright 2016-2018, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"os"
"github.com/pulumi/pulumi/pkg/backend/display"
"github.com/pulumi/pulumi/pkg/graph"
"github.com/pulumi/pulumi/pkg/graph/dotconv"
"github.com/pulumi/pulumi/pkg/resource"
"github.com/pulumi/pulumi/pkg/resource/deploy"
"github.com/pulumi/pulumi/pkg/util/cmdutil"
"github.com/spf13/cobra"
)
// Whether or not we should ignore parent edges when building up our graph.
var ignoreParentEdges bool
// Whether or not we should ignore dependency edges when building up our graph.
var ignoreDependencyEdges bool
// The color of dependency edges in the graph. Defaults to #246C60, a blush-green.
var dependencyEdgeColor string
// The color of parent edges in the graph. Defaults to #AA6639, an orange.
var parentEdgeColor string
func newStackGraphCmd() *cobra.Command {
var stackName string
cmd := &cobra.Command{
Use: "graph",
Args: cmdutil.ExactArgs(1),
Short: "Export a stack's dependency graph to a file",
Long: "Export a stack's dependency graph to a file.\n" +
"\n" +
"This command can be used to view the dependency graph that a Pulumi program\n" +
"admitted when it was ran. This graph is output in the DOT format. This command operates\n" +
"on your stack's most recent deployment.",
Run: cmdutil.RunFunc(func(cmd *cobra.Command, args []string) error {
opts := display.Options{
Color: cmdutil.GetGlobalColorization(),
}
Initial support for passing URLs to `new` and `up` (#1727) * Initial support for passing URLs to `new` and `up` This PR adds initial support for `pulumi new` using Git under the covers to manage Pulumi templates, providing the same experience as before. You can now also optionally pass a URL to a Git repository, e.g. `pulumi new [<url>]`, including subdirectories within the repository, and arbitrary branches, tags, or commits. The following commands result in the same behavior from the user's perspective: - `pulumi new javascript` - `pulumi new https://github.com/pulumi/templates/templates/javascript` - `pulumi new https://github.com/pulumi/templates/tree/master/templates/javascript` - `pulumi new https://github.com/pulumi/templates/tree/HEAD/templates/javascript` To specify an arbitrary branch, tag, or commit: - `pulumi new https://github.com/pulumi/templates/tree/<branch>/templates/javascript` - `pulumi new https://github.com/pulumi/templates/tree/<tag>/templates/javascript` - `pulumi new https://github.com/pulumi/templates/tree/<commit>/templates/javascript` Branches and tags can include '/' separators, and `pulumi` will still find the right subdirectory. URLs to Gists are also supported, e.g.: `pulumi new https://gist.github.com/justinvp/6673959ceb9d2ac5a14c6d536cb871a6` If the specified subdirectory in the repository does not contain a `Pulumi.yaml`, it will look for subdirectories within containing `Pulumi.yaml` files, and prompt the user to choose a template, along the lines of how `pulumi new` behaves when no template is specified. The following commands result in the CLI prompting to choose a template: - `pulumi new` - `pulumi new https://github.com/pulumi/templates/templates` - `pulumi new https://github.com/pulumi/templates/tree/master/templates` - `pulumi new https://github.com/pulumi/templates/tree/HEAD/templates` Of course, arbitrary branches, tags, or commits can be specified as well: - `pulumi new https://github.com/pulumi/templates/tree/<branch>/templates` - `pulumi new https://github.com/pulumi/templates/tree/<tag>/templates` - `pulumi new https://github.com/pulumi/templates/tree/<commit>/templates` This PR also includes initial support for passing URLs to `pulumi up`, providing a streamlined way to deploy installable cloud applications with Pulumi, without having to manage source code locally before doing a deployment. For example, `pulumi up https://github.com/justinvp/aws` can be used to deploy a sample AWS app. The stack can be updated with different versions, e.g. `pulumi up https://github.com/justinvp/aws/tree/v2 -s <stack-to-update>` Config values can optionally be passed via command line flags, e.g. `pulumi up https://github.com/justinvp/aws -c aws:region=us-west-2 -c foo:bar=blah` Gists can also be used, e.g. `pulumi up https://gist.github.com/justinvp/62fde0463f243fcb49f5a7222e51bc76` * Fix panic when hitting ^C from "choose template" prompt * Add description to templates When running `pulumi new` without specifying a template, include the template description along with the name in the "choose template" display. ``` $ pulumi new Please choose a template: aws-go A minimal AWS Go program aws-javascript A minimal AWS JavaScript program aws-python A minimal AWS Python program aws-typescript A minimal AWS TypeScript program > go A minimal Go program hello-aws-javascript A simple AWS serverless JavaScript program javascript A minimal JavaScript program python A minimal Python program typescript A minimal TypeScript program ``` * React to changes to the pulumi/templates repo. We restructured the `pulumi/templates` repo to have all the templates in the root instead of in a `templates` subdirectory, so make the change here to no longer look for templates in `templates`. This also fixes an issue around using `Depth: 1` that I found while testing this. When a named template is used, we attempt to clone or pull from the `pulumi/templates` repo to `~/.pulumi/templates`. Having it go in this well-known directory allows us to maintain previous behavior around allowing offline use of templates. If we use `Depth: 1` for the initial clone, it will fail when attempting to pull when there are updates to the remote repository. Unfortunately, there's no built-in `--unshallow` support in `go-git` and setting a larger `Depth` doesn't appear to help. There may be a workaround, but for now, if we're cloning the pulumi templates directory to `~/.pulumi/templates`, we won't use `Depth: 1`. For template URLs, we will continue to use `Depth: 1` as we clone those to a temp directory (which gets deleted) that we'll never try to update. * List available templates in help text * Address PR Feedback * Don't show "Installing dependencies" message for `up` * Fix secrets handling When prompting for config, if the existing stack value is a secret, keep it a secret and mask the prompt. If the template says it should be secret, make it a secret. * Fix ${PROJECT} and ${DESCRIPTION} handling for `up` Templates used with `up` should already have a filled-in project name and description, but if it's a `new`-style template, that has `${PROJECT}` and/or `${DESCRIPTION}`, be helpful and just replace these with better values. * Fix stack handling Add a bool `setCurrent` param to `requireStack` to control whether the current stack should be saved in workspace settings. For the `up <url>` case, we don't want to save. Also, split the `up` code into two separate functions: one for the `up <url>` case and another for the normal `up` case where you have workspace in your current directory. While we may be able to combine them back into a single function, right now it's a bit cleaner being separate, even with some small amount of duplication. * Fix panic due to nil crypter Lazily get the crypter only if needed inside `promptForConfig`. * Embellish comment * Harden isPreconfiguredEmptyStack check Fix the code to check to make sure the URL specified on the command line matches the URL stored in the `pulumi:template` config value, and that the rest of the config from the stack satisfies the config requirements of the template.
2018-08-11 03:08:16 +02:00
s, err := requireStack(stackName, false, opts, true /*setCurrent*/)
if err != nil {
return err
}
Show manifest information for stacks This change supports displaying manifest information for a stack and changes the way we handle Snapshots in our backend. Previously, every call to GetStack would synthesize a Snapshot by taking the set of resources returned from the `/api/stacks/<owner>/<name>` endpoint, combined with an empty manfiest (since the service was not returning the manifest). This wasn't great for two reasons: 1. We didn't have manifest information, so we couldn't display any of its information (most important the last updated time). 2. This strategy required that the service return all the resources for a stack anytime GetStack was called. While the CLI did not often need this detailed information the fact that we forced the Service to produce it (which in the case of stack managed PPC would require the service to talk to yet another service) creates a bunch of work that we end up ignoring. I've refactored the code such that `backend.Stack`'s `Snapshot()` method now lazily requests the information from the service such that we can construct a `Snapshot()` on demand and only pay the cost when we actually need it. I think making more of this stuff lazy is the long term direction we want to follow. Unfortunately, right now, it means in cases where we do need this data we end up fetching it twice. The service does it once when we call GetStack and then we do it again when we actually need to get at the Snapshot. However, once we land this change, we can update the service to no longer return resources on the apistack.Stack type. The CLI no longer needs this property. We'll likely want to continue in a direction where `apistack.Stack` can be created quickly by the service (without expensive database queries or fetching remote resources) and just add additional endpoints that let us get at the specific information we want in the specific cases when we want it instead of forcing us to return a bunch of data that we often ignore. Fixes pulumi/pulumi-service#371
2018-05-23 00:39:13 +02:00
snap, err := s.Snapshot(commandContext())
if err != nil {
return err
}
Show manifest information for stacks This change supports displaying manifest information for a stack and changes the way we handle Snapshots in our backend. Previously, every call to GetStack would synthesize a Snapshot by taking the set of resources returned from the `/api/stacks/<owner>/<name>` endpoint, combined with an empty manfiest (since the service was not returning the manifest). This wasn't great for two reasons: 1. We didn't have manifest information, so we couldn't display any of its information (most important the last updated time). 2. This strategy required that the service return all the resources for a stack anytime GetStack was called. While the CLI did not often need this detailed information the fact that we forced the Service to produce it (which in the case of stack managed PPC would require the service to talk to yet another service) creates a bunch of work that we end up ignoring. I've refactored the code such that `backend.Stack`'s `Snapshot()` method now lazily requests the information from the service such that we can construct a `Snapshot()` on demand and only pay the cost when we actually need it. I think making more of this stuff lazy is the long term direction we want to follow. Unfortunately, right now, it means in cases where we do need this data we end up fetching it twice. The service does it once when we call GetStack and then we do it again when we actually need to get at the Snapshot. However, once we land this change, we can update the service to no longer return resources on the apistack.Stack type. The CLI no longer needs this property. We'll likely want to continue in a direction where `apistack.Stack` can be created quickly by the service (without expensive database queries or fetching remote resources) and just add additional endpoints that let us get at the specific information we want in the specific cases when we want it instead of forcing us to return a bunch of data that we often ignore. Fixes pulumi/pulumi-service#371
2018-05-23 00:39:13 +02:00
dg := makeDependencyGraph(snap)
file, err := os.Create(args[0])
if err != nil {
return err
}
if err := dotconv.Print(dg, file); err != nil {
_ = file.Close()
return err
}
cmd.Printf("%sWrote stack dependency graph to `%s`", cmdutil.EmojiOr("🔍 ", ""), args[0])
cmd.Println()
return file.Close()
}),
}
cmd.PersistentFlags().StringVarP(
&stackName, "stack", "s", "", "The name of the stack to operate on. Defaults to the current stack")
cmd.PersistentFlags().BoolVar(&ignoreParentEdges, "ignore-parent-edges", false,
"Ignores edges introduced by parent/child resource relationships")
cmd.PersistentFlags().BoolVar(&ignoreDependencyEdges, "ignore-dependency-edges", false,
"Ignores edges introduced by dependency resource relationships")
cmd.PersistentFlags().StringVar(&dependencyEdgeColor, "dependency-edge-color", "#246C60",
"Sets the color of dependency edges in the graph")
cmd.PersistentFlags().StringVar(&parentEdgeColor, "parent-edge-color", "#AA6639",
"Sets the color of parent edges in the graph")
return cmd
}
// All of the types and code within this file are to provide implementations of the interfaces
// in the `graph` package, so that we can use the `dotconv` package to output our graph in the
// DOT format.
//
// `dependencyEdge` implements graph.Edge, `dependencyVertex` implements graph.Vertex, and
// `dependencyGraph` implements `graph.Graph`.
type dependencyEdge struct {
to *dependencyVertex
from *dependencyVertex
}
// In this simple case, edges have no data.
func (edge *dependencyEdge) Data() interface{} {
return nil
}
// In this simple case, edges have no label.
func (edge *dependencyEdge) Label() string {
return ""
}
func (edge *dependencyEdge) To() graph.Vertex {
return edge.to
}
func (edge *dependencyEdge) From() graph.Vertex {
return edge.from
}
func (edge *dependencyEdge) Color() string {
return dependencyEdgeColor
}
// parentEdges represent edges in the parent-child graph, which
// exists alongside the dependency graph. An edge exists from node
// A to node B if node B is considered to be a parent of node A.
type parentEdge struct {
to *dependencyVertex
from *dependencyVertex
}
func (edge *parentEdge) Data() interface{} {
return nil
}
// In this simple case, edges have no label.
func (edge *parentEdge) Label() string {
return ""
}
func (edge *parentEdge) To() graph.Vertex {
return edge.to
}
func (edge *parentEdge) From() graph.Vertex {
return edge.from
}
func (edge *parentEdge) Color() string {
return parentEdgeColor
}
// A dependencyVertex contains a reference to the graph to which it belongs
// and to the resource state that it represents. Incoming and outgoing edges
// are calculated on-demand using the combination of the graph and the state.
type dependencyVertex struct {
graph *dependencyGraph
resource *resource.State
incomingEdges []graph.Edge
outgoingEdges []graph.Edge
}
func (vertex *dependencyVertex) Data() interface{} {
return vertex.resource
}
func (vertex *dependencyVertex) Label() string {
return string(vertex.resource.URN)
}
func (vertex *dependencyVertex) Ins() []graph.Edge {
return vertex.incomingEdges
}
// Outgoing edges are indirectly calculated by traversing the entire graph looking
// for edges that point to this vertex. This is slow, but our graphs aren't big enough
// for this to matter too much.
func (vertex *dependencyVertex) Outs() []graph.Edge {
return vertex.outgoingEdges
}
// A dependencyGraph is a thin wrapper around a map of URNs to vertices in
// the graph. It is constructed directly from a snapshot.
type dependencyGraph struct {
vertices map[resource.URN]*dependencyVertex
}
// Roots are edges that point to the root set of our graph. In our case,
// for simplicity, we define the root set of our dependency graph to be everything.
func (dg *dependencyGraph) Roots() []graph.Edge {
rootEdges := []graph.Edge{}
for _, vertex := range dg.vertices {
edge := &dependencyEdge{
to: vertex,
from: nil,
}
rootEdges = append(rootEdges, edge)
}
return rootEdges
}
// Makes a dependency graph from a deployment snapshot, allocating a vertex
// for every resource in the graph.
func makeDependencyGraph(snapshot *deploy.Snapshot) *dependencyGraph {
dg := &dependencyGraph{
vertices: make(map[resource.URN]*dependencyVertex),
}
for _, resource := range snapshot.Resources {
vertex := &dependencyVertex{
graph: dg,
resource: resource,
}
dg.vertices[resource.URN] = vertex
}
for _, vertex := range dg.vertices {
if !ignoreDependencyEdges {
// Incoming edges are directly stored within the checkpoint file; they represent
// resources on which this vertex immediately depends upon.
for _, dep := range vertex.resource.Dependencies {
vertexWeDependOn := vertex.graph.vertices[dep]
edge := &dependencyEdge{to: vertex, from: vertexWeDependOn}
vertex.incomingEdges = append(vertex.incomingEdges, edge)
vertexWeDependOn.outgoingEdges = append(vertexWeDependOn.outgoingEdges, edge)
}
}
// alongside the dependency graph sits the resource parentage graph, which
// is also displayed as part of this graph, although with different colored
// edges.
if !ignoreParentEdges {
if parent := vertex.resource.Parent; parent != resource.URN("") {
parentVertex := dg.vertices[parent]
vertex.outgoingEdges = append(vertex.outgoingEdges, &parentEdge{
to: parentVertex,
from: vertex,
})
}
}
}
return dg
}