Store stack specific config in folder named after owner

When the backend for a stack is the Pulumi Service, store
configuration values for the stack in a folder with the name of the
owner.

This means that `ellismg/my-great-stack` and `pulumi/my-great-stack`
now have different configuration files, instead of sharing the
`Pulumi.my-great-stack.yaml` file stored in the configuration root
directory. Since these are two different deployments, it makes sense
that they can be configured independently. In addition, since
encrypted configuration values are stored using a per stack key,
sharing the files doesn't actually work when using encrypted
configuration.

For compat, the CLI prefers the older file name when present.

Fixes #1859

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 - Added a `--stack` argument (short form `-s`) to `pulumi stack`, `pulumi stack init`, `pulumi state delete` and `pulumi state unprotect` to allow operating on a different stack than the currently selected stack. This brings these commands in line with the other commands that operate on stacks and already provided a `--stack` option (fixes [pulumi/pulumi#1648](https://github.com/pulumi/pulumi/issues/1648))
 
+- Stack specific configuration settings are now stored in yaml files under the `.pulumi` folder, next to `Pulumi.yaml` instead of being in the same directory as `Pulumi.yaml`. However, if a configuration file is in the old location, it is used instead. The `config` property of the project's `Pulumi.yaml` can pick a different directory. To go back to the old behavior, add `config: .` to `Pulumi.yaml` (fixes [pulumi/pulumi#2005](https://github.com/pulumi/pulumi/issues/2005))
+
+- When using the Pulumi Service, stack configuration files are now segmented by owner. Previously, we share the same on disk location for a stack if its name was shared across organizations. This would lead to issues when encrypted configuration (which was encrypted with a per stack key) was not usable, except for the organization that the stack was created in. If an existing configuration file exists at the old location, it is prefered, to match behavior with older CLIs (fixes), so when you want to take advantage of this new feature, we recommend that you either manually move the old `Pulumi.<stack-name>.yaml` into its new prefered location (by default: `.pulumi/<organization-name>/Pulumi.<stack-name>.yaml`) or remove it from disk and run `pulumi config refresh` to sync the configuration from the last deployment into a new file in the correct location.  (fixes [pulumi/pulumi#1859](https://github.com/pulumi/pulumi/issues/1859))
+
 ## 0.16.7 (Release December 5th, 2018)
 
 ### Improvements

cmd/config.go

@@ -310,21 +310,21 @@ var stackConfigFile string
 
 func getProjectStackPath(stack backend.Stack) (string, error) {
 	if stackConfigFile == "" {
-		return workspace.DetectProjectStackPath(stack.Ref().Name())
+		return workspace.DetectProjectStackPath(stack.Ref().Owner(), stack.Ref().Name())
 	}
 	return stackConfigFile, nil
 }
 
 func loadProjectStack(stack backend.Stack) (*workspace.ProjectStack, error) {
 	if stackConfigFile == "" {
-		return workspace.DetectProjectStack(stack.Ref().Name())
+		return workspace.DetectProjectStack(stack.Ref().Owner(), stack.Ref().Name())
 	}
 	return workspace.LoadProjectStack(stackConfigFile)
 }
 
 func saveProjectStack(stack backend.Stack, ps *workspace.ProjectStack) error {
 	if stackConfigFile == "" {
-		return workspace.SaveProjectStack(stack.Ref().Name(), ps)
+		return workspace.SaveProjectStack(stack.Ref().Owner(), stack.Ref().Name(), ps)
 	}
 	return ps.Save(stackConfigFile)
 }

cmd/stack_rm.go

@@ -79,7 +79,7 @@ func newStackRmCmd() *cobra.Command {
 
 			if !preserveConfig {
 				// Blow away stack specific settings if they exist. If we get an ENOENT error, ignore it.
-				if path, err := workspace.DetectProjectStackPath(s.Ref().Name()); err == nil {
+				if path, err := workspace.DetectProjectStackPath(s.Ref().Owner(), s.Ref().Name()); err == nil {
 					if err = os.Remove(path); err != nil && !os.IsNotExist(err) {
 						return err
 					}

pkg/backend/backend.go

@@ -56,6 +56,9 @@ func (e StackAlreadyExistsError) Error() string {
 type StackReference interface {
 	// fmt.Stringer's String() method returns a string of the stack identity, suitable for display in the CLI
 	fmt.Stringer
+	// Owner is the name of the account that owns this stack. Not all backends have the concept of an owner (e.g the
+	// local filebased backend has no such concept) in which case this function returns the empty string.
+	Owner() string
 	// Name is the name that will be passed to the Pulumi engine when preforming operations on this stack. This
 	// name may not uniquely identify the stack (e.g. the cloud backend embeds owner information in the StackReference
 	// but that informaion is not part of the StackName() we pass to the engine.

pkg/backend/filestate/backend.go

@@ -67,6 +67,10 @@ func (r localBackendReference) String() string {
 	return string(r.name)
 }
 
+func (r localBackendReference) Owner() string {
+	return ""
+}
+
 func (r localBackendReference) Name() tokens.QName {
 	return r.name
 }

pkg/backend/filestate/crypto.go

@@ -53,7 +53,7 @@ func symmetricCrypter(stackName tokens.QName, configFile string) (config.Crypter
 	contract.Assertf(stackName != "", "stackName %s", "!= \"\"")
 
 	if configFile == "" {
-		f, err := workspace.DetectProjectStackPath(stackName)
+		f, err := workspace.DetectProjectStackPath("", stackName)
 		if err != nil {
 			return nil, err
 		}

pkg/backend/filestate/state.go

@@ -88,7 +88,7 @@ func (b *localBackend) newUpdate(stackName tokens.QName, proj *workspace.Project
 func (b *localBackend) getTarget(stackName tokens.QName) (*deploy.Target, error) {
 	stackConfigFile := b.stackConfigFile
 	if stackConfigFile == "" {
-		f, err := workspace.DetectProjectStackPath(stackName)
+		f, err := workspace.DetectProjectStackPath("", stackName)
 		if err != nil {
 			return nil, err
 		}

pkg/backend/httpstate/backend.go

@@ -657,7 +657,7 @@ func (b *cloudBackend) createAndStartUpdate(
 	}
 	stackConfigFile := b.stackConfigFile
 	if stackConfigFile == "" {
-		f, err := workspace.DetectProjectStackPath(stackRef.Name())
+		f, err := workspace.DetectProjectStackPath(stackRef.Owner(), stackRef.Name())
 		if err != nil {
 			return client.UpdateIdentifier{}, 0, "", err
 		}

pkg/backend/httpstate/stack.go

@@ -56,6 +56,10 @@ func (c cloudBackendReference) String() string {
 	return fmt.Sprintf("%s/%s", c.owner, c.name)
 }
 
+func (c cloudBackendReference) Owner() string {
+	return c.owner
+}
+
 func (c cloudBackendReference) Name() tokens.QName {
 	return c.name
 }

pkg/backend/httpstate/state.go

@@ -250,7 +250,7 @@ func (b *cloudBackend) getTarget(ctx context.Context, stackRef backend.StackRefe
 	// Pull the local stack info so we can get at its configuration bag.
 	stackConfigFile := b.stackConfigFile
 	if stackConfigFile == "" {
-		f, err := workspace.DetectProjectStackPath(stackRef.Name())
+		f, err := workspace.DetectProjectStackPath(stackRef.Owner(), stackRef.Name())
 		if err != nil {
 			return nil, err
 		}

pkg/workspace/paths.go

@@ -73,16 +73,54 @@ func DetectProjectPath() (string, error) {
 	return path, nil
 }
 
-// DetectProjectStackPath returns the name of the file to store stack specific project settings in. We place stack
-// specific settings next to the Pulumi.yaml file, named like: Pulumi.<stack-name>.yaml
-func DetectProjectStackPath(stackName tokens.QName) (string, error) {
+// DetectProjectStackPath returns the name of the file to store stack specific project settings in. By default, we
+// write this into a .pulumi folder next to the Pulumi.yaml file for the project, and for stacks managed by the
+// pulumi service, where there is the concept of an "owner" we further segment these files into directories named after
+// the owner.
+//
+// The ".pulumi" folder may be configured by setting the `config` property in the Project's Pulumi.yaml file.
+//
+// For compatibility, we have a few other paths that we prefer (when an actual file exists on disk), but when no
+// existing configuration file is present, we use the above path.
+func DetectProjectStackPath(owner string, stackName tokens.QName) (string, error) {
 	proj, projPath, err := DetectProjectAndPath()
 	if err != nil {
 		return "", err
 	}
 
-	return filepath.Join(filepath.Dir(projPath), proj.Config, fmt.Sprintf("%s.%s%s", ProjectFile, qnameFileName(stackName),
-		filepath.Ext(projPath))), nil
+	projectRoot := filepath.Dir(projPath)
+	configRoot := proj.Config
+	stackFileName := qnameFileName(stackName)
+	stackFileExt := filepath.Ext(projPath)
+
+	// As our configuration system has evolved, we've made some changes to where we store stack specific configuration
+	// on disk. `candidates` is slice of possible paths to look. We start at the first element of the slice and return
+	// the first path that exists (so these are ordered from older formats to newer formats). If none of these files
+	// exist, we say the path is the last element in this slice (and so that should be the most preferred format).
+	candidates := []string{
+		filepath.Join(projectRoot, configRoot, fmt.Sprintf("%s.%s%s", ProjectFile, stackFileName, stackFileExt)),
+		filepath.Join(projectRoot, configRoot, owner, fmt.Sprintf("%s.%s%s", ProjectFile, stackFileName, stackFileExt)),
+	}
+
+	// When configRoot is unset, we also include ".pulumi" at the end of our candiates lists. We do this because we'd
+	// like an unset configuration root to mean ".pulumi", a change in behavior from the old default where we would
+	// just write them into next to Pulumi.yaml.
+	if configRoot == "" {
+		candidates = append(candidates, filepath.Join(projectRoot, ".pulumi",
+			fmt.Sprintf("%s.%s%s", ProjectFile, stackFileName, stackFileExt)))
+		candidates = append(candidates, filepath.Join(projectRoot, ".pulumi", owner,
+			fmt.Sprintf("%s.%s%s", ProjectFile, stackFileName, stackFileExt)))
+	}
+
+	for _, candidate := range candidates {
+		if _, err := os.Stat(candidate); err == nil {
+			return candidate, nil
+		}
+	}
+
+	// In the case where none of the candidate file exists, the last one in the candidate list is the canonical path
+	// we'd prefer to use.
+	return candidates[len(candidates)-1], nil
 }
 
 // DetectProjectPathFrom locates the closest project from the given path, searching "upwards" in the directory
@@ -99,8 +137,8 @@ func DetectProject() (*Project, error) {
 	return proj, err
 }
 
-func DetectProjectStack(stackName tokens.QName) (*ProjectStack, error) {
-	path, err := DetectProjectStackPath(stackName)
+func DetectProjectStack(owner string, stackName tokens.QName) (*ProjectStack, error) {
+	path, err := DetectProjectStackPath(owner, stackName)
 	if err != nil {
 		return nil, err
 	}
@@ -132,8 +170,8 @@ func SaveProject(proj *Project) error {
 	return proj.Save(path)
 }
 
-func SaveProjectStack(stackName tokens.QName, stack *ProjectStack) error {
-	path, err := DetectProjectStackPath(stackName)
+func SaveProjectStack(owner string, stackName tokens.QName, stack *ProjectStack) error {
+	path, err := DetectProjectStackPath(owner, stackName)
 	if err != nil {
 		return err
 	}

tests/integration/integration_test.go

@@ -411,9 +411,9 @@ func TestConfigSave(t *testing.T) {
 		assert.Equal(t, v, dv)
 	}
 
-	testStack1, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-1.yaml"))
+	testStack1, err := workspace.LoadProjectStack(filepath.Join(e.CWD, ".pulumi", "Pulumi.testing-1.yaml"))
 	assert.NoError(t, err)
-	testStack2, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-2.yaml"))
+	testStack2, err := workspace.LoadProjectStack(filepath.Join(e.CWD, ".pulumi", "Pulumi.testing-2.yaml"))
 	assert.NoError(t, err)
 
 	assert.Equal(t, 2, len(testStack1.Config))