maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Load default providers deterministically (#2590)

Browse source 
* Load default providers deterministically

This commit adds a new algorithm for deriving a list of default
providers from the set of plugins reported from the language host and
from the snapshot. If the language host reports a set of plugins,
default providers are sourced directly from that set, otherwise default
providers are sourced from the full set of plugins, including ones from
the snapshot.

When multiple versions of the same provider are requested, the newest
version of that provider is always select as the default provider.

* Add CHANGELOG.md entry

* Skip the language host's plugins if it reports no resource plugins

* CR feedback

* CR: Log when skipping non resource plugin
This commit is contained in:
Sean Gillespie 2019-03-26 13:29:34 -07:00 committed by GitHub
parent 3324dc3249
commit 4d227f7ed2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 235 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -7,6 +7,10 @@
- A new command, `pulumi stack rename` was added. This allows you to change the name of an existing stack in a project. Note: When a stack is renamed, the `pulumi.getStack` function in the SDK will now return a new value. If a stack name is used as part of a resource name, the next `pulumi update` will not understand that the old and new resources are logically the same. We plan to support adding aliases to individual resources so you can handle these cases. See [pulumi/pulumi#458](https://github.com/pulumi/pulumi/issues/458) for discussion on this new feature. For now, if you are unwilling to have `pulumi update` create and destroy these resources, you can rename your stack back to the old name. (fixes [pulumi/pulumi#2402](https://github.com/pulumi/pulumi/issues/2402))
- Fix two warnings that were printed when using a dynamic provider about missing method handlers.
- A bug in the previous version of the Pulumi CLI occasionally caused the Pulumi Engine to load the incorrect resource
plugin when processing an update. This bug has been fixed in 0.17.3 by performing a deterministic selection of the
best set of plugins available to the engine before starting up. See
[pulumi/pulumi#2579](https://github.com/pulumi/pulumi/issues/2579) for discussion on this issue.
## 0.17.2 (Released March 15, 2019)

95
pkg/engine/plugins.go
View file

@ -16,12 +16,15 @@ package engine
import (
"context"
"sort"
"github.com/blang/semver"
"golang.org/x/sync/errgroup"
"github.com/pulumi/pulumi/pkg/resource/deploy"
"github.com/pulumi/pulumi/pkg/resource/deploy/providers"
"github.com/pulumi/pulumi/pkg/resource/plugin"
"github.com/pulumi/pulumi/pkg/tokens"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/logging"
"github.com/pulumi/pulumi/pkg/workspace"
@ -183,3 +186,95 @@ func installPlugin(client deploy.BackendClient, plugin workspace.PluginInfo) err
logging.V(7).Infof("installPlugin(%s, %s): successfully installed", plugin.Name, plugin.Version)
return nil
}
// computeDefaultProviderPlugins computes, for every resource plugin, a mapping from packages to semver versions
// reflecting the version of a provider that should be used as the "default" resource when registering resources. This
// function takes two sets of plugins: a set of plugins given to us from the language host and the full set of plugins.
// If the language host has sent us a non-empty set of plugins, we will use those exclusively to service default
// provider requests. Otherwise, we will use the full set of plugins, which is the existing behavior today.
//
// The justification for favoring language plugins over all else is that, ultimately, it is the language plugin that
// produces resource registrations and therefore it is the language plugin that should dictate exactly what plugins to
// use to satisfy a resource registration. Since we do not today request a particular version of a plugin via
// RegisterResource (pulumi/pulumi#2389), this is the best we can do to infer the version that the language plugin
// actually wants.
//
// Whenever a resource arrives via RegisterResource and does not explicitly specify which provider to use, the engine
// injects a "default" provider resource that will serve as that resource's provider. This function computes the map
// that the engine uses to determine which version of a particular provider to load.
//
// it is critical that this function be 100% deterministic.
func computeDefaultProviderPlugins(languagePlugins, allPlugins pluginSet) map[tokens.Package]*semver.Version {
// Language hosts are not required to specify the full set of plugins they depend on. If the set of plugins received
// from the language host does not include any resource providers, fall back to the full set of plugins.
languageReportedProviderPlugins := false
for _, plug := range languagePlugins.Values() {
if plug.Kind == workspace.ResourcePlugin {
languageReportedProviderPlugins = true
}
}
sourceSet := languagePlugins
if !languageReportedProviderPlugins {
logging.V(preparePluginLog).Infoln(
"computeDefaultProviderPlugins(): language host reported empty set of provider plugins, using all plugins")
sourceSet = allPlugins
}
defaultProviderVersions := make(map[tokens.Package]*semver.Version)
// Sort the set of source plugins by version, so that we iterate over the set of plugins in a deterministic order.
// Sorting by version gets us two properties:
// 1. The below loop will never see a nil-versioned plugin after a non-nil versioned plugin, since the sort always
// considers nil-versioned plugins to be less than non-nil versioned plugins.
// 2. The below loop will never see a plugin with a version that is older than a plugin that has already been
// seen. The sort will always have placed the older plugin before the newer plugin.
//
// Despite these properties, the below loop explicitly handles those cases to preserve correct behavior even if the
// sort is not functioning properly.
sourcePlugins := sourceSet.Values()
sort.Sort(workspace.SortedPluginInfo(sourcePlugins))
for _, p := range sourcePlugins {
logging.V(preparePluginLog).Infof("computeDefaultProviderPlugins(): considering %s", p)
if p.Kind != workspace.ResourcePlugin {
// Default providers are only relevant for resource plugins.
logging.V(preparePluginVerboseLog).Infof(
"computeDefaultProviderPlugins(): skipping %s, not a resource provider", p)
continue
}
if seenVersion, has := defaultProviderVersions[tokens.Package(p.Name)]; has {
if seenVersion == nil {
logging.V(preparePluginLog).Infof(
"computeDefaultProviderPlugins(): plugin %s selected for package %s (override, previous was nil)",
p, p.Name)
defaultProviderVersions[tokens.Package(p.Name)] = p.Version
continue
}
contract.Assertf(p.Version != nil, "p.Version should not be nil if sorting is correct!")
if p.Version != nil && p.Version.GT(*seenVersion) {
logging.V(preparePluginLog).Infof(
"computeDefaultProviderPlugins(): plugin %s selected for package %s (override, newer than previous %s)",
p, p.Name, seenVersion)
defaultProviderVersions[tokens.Package(p.Name)] = p.Version
continue
}
contract.Failf("Should not have seen an older plugin if sorting is correct!")
}
logging.V(preparePluginLog).Infof(
"computeDefaultProviderPlugins(): plugin %s selected for package %s (first seen)", p, p.Name)
defaultProviderVersions[tokens.Package(p.Name)] = p.Version
}
if logging.V(preparePluginLog) {
logging.V(preparePluginLog).Infoln("computeDefaultProviderPlugins(): summary of default plugins:")
for pkg, version := range defaultProviderVersions {
logging.V(preparePluginLog).Infof(" %-15s = %s", pkg, version)
}
}
return defaultProviderVersions
}

129
pkg/engine/plugins_test.go Normal file
View file

@ -0,0 +1,129 @@
// Copyright 2016-2019, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package engine
import (
"testing"
"github.com/blang/semver"
"github.com/stretchr/testify/assert"
"github.com/pulumi/pulumi/pkg/tokens"
"github.com/pulumi/pulumi/pkg/util/logging"
"github.com/pulumi/pulumi/pkg/workspace"
)
func mustMakeVersion(v string) *semver.Version {
ver := semver.MustParse(v)
return &ver
}
func TestDefaultProvidersSingle(t *testing.T) {
logging.InitLogging(true, 7, false)
languagePlugins := newPluginSet()
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.1"),
Kind: workspace.ResourcePlugin,
})
languagePlugins.Add(workspace.PluginInfo{
Name: "kubernetes",
Version: mustMakeVersion("0.22.0"),
Kind: workspace.ResourcePlugin,
})
defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
assert.NotNil(t, defaultProviders)
awsVer, ok := defaultProviders[tokens.Package("aws")]
assert.True(t, ok)
assert.NotNil(t, awsVer)
assert.Equal(t, "0.17.1", awsVer.String())
kubernetesVer, ok := defaultProviders[tokens.Package("kubernetes")]
assert.True(t, ok)
assert.NotNil(t, kubernetesVer)
assert.Equal(t, "0.22.0", kubernetesVer.String())
}
func TestDefaultProvidersOverrideNoVersion(t *testing.T) {
logging.InitLogging(true, 7, false)
languagePlugins := newPluginSet()
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.1"),
Kind: workspace.ResourcePlugin,
})
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: nil,
Kind: workspace.ResourcePlugin,
})
defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
assert.NotNil(t, defaultProviders)
awsVer, ok := defaultProviders[tokens.Package("aws")]
assert.True(t, ok)
assert.NotNil(t, awsVer)
assert.Equal(t, "0.17.1", awsVer.String())
}
func TestDefaultProvidersOverrideNewerVersion(t *testing.T) {
languagePlugins := newPluginSet()
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.0"),
Kind: workspace.ResourcePlugin,
})
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.1"),
Kind: workspace.ResourcePlugin,
})
languagePlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.2-dev.1553126336"),
Kind: workspace.ResourcePlugin,
})
defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
assert.NotNil(t, defaultProviders)
awsVer, ok := defaultProviders[tokens.Package("aws")]
assert.True(t, ok)
assert.NotNil(t, awsVer)
assert.Equal(t, "0.17.2-dev.1553126336", awsVer.String())
}
func TestDefaultProvidersSnapshotOverrides(t *testing.T) {
languagePlugins := newPluginSet()
languagePlugins.Add(workspace.PluginInfo{
Name: "python",
Kind: workspace.LanguagePlugin,
})
snapshotPlugins := newPluginSet()
snapshotPlugins.Add(workspace.PluginInfo{
Name: "aws",
Version: mustMakeVersion("0.17.0"),
Kind: workspace.ResourcePlugin,
})
defaultProviders := computeDefaultProviderPlugins(languagePlugins, snapshotPlugins)
assert.NotNil(t, defaultProviders)
awsVer, ok := defaultProviders[tokens.Package("aws")]
assert.True(t, ok)
assert.NotNil(t, awsVer)
assert.Equal(t, "0.17.0", awsVer.String())
}

11
pkg/engine/update.go
View file

@ -18,13 +18,10 @@ import (
"sync"
"time"
"github.com/blang/semver"
"github.com/pulumi/pulumi/pkg/diag"
"github.com/pulumi/pulumi/pkg/resource"
"github.com/pulumi/pulumi/pkg/resource/deploy"
"github.com/pulumi/pulumi/pkg/resource/plugin"
"github.com/pulumi/pulumi/pkg/tokens"
"github.com/pulumi/pulumi/pkg/util/contract"
"github.com/pulumi/pulumi/pkg/util/logging"
"github.com/pulumi/pulumi/pkg/util/result"
@ -143,13 +140,7 @@ func newUpdateSource(
}
// Collect the version information for default providers.
defaultProviderVersions := make(map[tokens.Package]*semver.Version)
for _, p := range allPlugins.Values() {
if p.Kind != workspace.ResourcePlugin {
continue
}
defaultProviderVersions[tokens.Package(p.Name)] = p.Version
}
defaultProviderVersions := computeDefaultProviderPlugins(languagePlugins, allPlugins)
// If that succeeded, create a new source that will perform interpretation of the compiled program.
// TODO[pulumi/pulumi#88]: we are passing `nil` as the arguments map; we need to allow a way to pass these.

11
pkg/workspace/plugins.go
View file

@ -462,10 +462,11 @@ func GetPluginPath(kind PluginKind, name string, version *semver.Version) (strin
return "", "", nil
}
type sortedPluginInfo []PluginInfo
// SortedPluginInfo is a wrapper around PluginInfo that allows for sorting by version.
type SortedPluginInfo []PluginInfo
func (sp sortedPluginInfo) Len() int { return len(sp) }
func (sp sortedPluginInfo) Less(i, j int) bool {
func (sp SortedPluginInfo) Len() int { return len(sp) }
func (sp SortedPluginInfo) Less(i, j int) bool {
iVersion := sp[i].Version
jVersion := sp[j].Version
switch {
@ -479,7 +480,7 @@ func (sp sortedPluginInfo) Less(i, j int) bool {
return iVersion.LT(*jVersion)
}
}
func (sp sortedPluginInfo) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] }
func (sp SortedPluginInfo) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] }
// SelectCompatiblePlugin selects a plugin from the list of plugins with the given kind and name that sastisfies the
// requested semver range. It returns the highest version plugin that satisfies the requested constraints, or an error
@ -499,7 +500,7 @@ func SelectCompatiblePlugin(
//
// Plugins without versions are treated as having the lowest version. Ties between plugins without versions are
// resolved arbitrarily.
sort.Sort(sortedPluginInfo(plugins))
sort.Sort(SortedPluginInfo(plugins))
for _, plugin := range plugins {
switch {
case plugin.Kind != kind || plugin.Name != name: