Reimplement getRequiredPlugins for go sdk (#4297)

This commit is contained in:
Evan Boyle 2020-04-06 12:30:40 -07:00 committed by GitHub
parent 4862d73186
commit ab659aa0c1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 212 additions and 18 deletions

View file

@ -17,6 +17,9 @@ CHANGELOG
- Fix handling of secret values in mock-based tests.
[#4272](https://github.com/pulumi/pulumi/pull/4272)
- Automatic plugin acquisition for Go
[#4297](https://github.com/pulumi/pulumi/pull/4297)
## 1.14.0 (2020-04-01)
- Fix error related to side-by-side versions of `@pulumi/pulumi`.
[#4235](https://github.com/pulumi/pulumi/pull/4235)

View file

@ -994,12 +994,6 @@ func (pkg *pkgContext) genConfig(w io.Writer, variables []*schema.Property) erro
return nil
}
func (pkg *pkgContext) genPackageRegistration(w io.Writer) {
fmt.Fprintf(w, "func init() {\n")
fmt.Fprintf(w, "\tpulumi.RegisterPackage(pulumi.PackageInfo{Name:\"%s\", Version:\"%s\"})\n", pkg.pkg.Name, pkg.pkg.Version.String())
fmt.Fprintf(w, "}\n")
}
func (pkg *pkgContext) getPkg(mod string) *pkgContext {
if override, ok := pkg.modToPkg[mod]; ok {
mod = override
@ -1172,7 +1166,7 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
files[relPath] = formattedSource
}
name, registerPackage := pkg.Name, pkg.Provider != nil
name := pkg.Name
for _, mod := range pkgMods {
pkg := packages[mod]
@ -1249,16 +1243,6 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
setFile(path.Join(mod, "pulumiTypes.go"), buffer.String())
}
// Package registration
if registerPackage {
buffer := &bytes.Buffer{}
pkg.genHeader(buffer, []string{"github.com/pulumi/pulumi/sdk/go/pulumi"}, nil)
pkg.genPackageRegistration(buffer)
setFile(path.Join(mod, "pulumiManifest.go"), buffer.String())
}
// Utilities
if pkg.needsUtils {
buffer := &bytes.Buffer{}

View file

@ -154,11 +154,14 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -199,6 +202,9 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

View file

@ -29,6 +29,7 @@ require (
github.com/uber/jaeger-client-go v2.22.1+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
golang.org/x/mod v0.2.0
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d
google.golang.org/grpc v1.28.0

View file

@ -25,6 +25,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@ -194,12 +195,15 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww=
golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -243,6 +247,9 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

View file

@ -19,9 +19,11 @@ import (
"fmt"
"io"
"regexp"
"strings"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/sdk/go/common/util/contract"
"golang.org/x/mod/semver"
)
var (
@ -35,8 +37,18 @@ var (
`^v(?P<version>\d+\.\d+\.\d+)-alpha\.(?P<time>\d+)\+(?P<gitInfo>g[a-z0-9]+)(?P<dirty>.dirty)?$`)
devVersionRegex = regexp.MustCompile(
`^v(?P<version>\d+\.\d+\.\d+)-dev\.(?P<time>\d+)\+(?P<gitInfo>g[a-z0-9]+)(?P<dirty>.dirty)?$`)
// nolint: lll
// https://github.com/golang/go/blob/9f40f9f4d3e9e5a08cfd1df5af23a6f61d67d408/src/cmd/go/internal/modfetch/pseudo.go#L49
pseudoVersionRegex = regexp.MustCompile(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
)
// IsPseudoVersion reports whether v is a go modules pseudo-version.
// nolint: lll
// https://github.com/golang/go/blob/9f40f9f4d3e9e5a08cfd1df5af23a6f61d67d408/src/cmd/go/internal/modfetch/pseudo.go#L118
func IsPseudoVersion(v string) bool {
return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRegex.MatchString(v)
}
// PyPiVersionFromNpmVersion returns a PEP-440 compliant version for a given semver version. This method does not
// support all possible semver strings, but instead just supports versions that we generate for our node packages.
//

View file

@ -36,3 +36,14 @@ func TestVersions(t *testing.T) {
assert.Equal(t, expected, p, "failed parsing '%s'", ver)
}
}
func TestPseduoVersion(t *testing.T) {
pseudoVersion := "v1.29.1-0.20200403140640-efb5e2a48a86"
assert.True(t, IsPseudoVersion(pseudoVersion))
tagVersion := "v1.29.0"
assert.False(t, IsPseudoVersion(tagVersion))
betaVersion := "v1.29.0-beta.1"
assert.False(t, IsPseudoVersion(betaVersion))
}

View file

@ -15,20 +15,25 @@
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"github.com/blang/semver"
pbempty "github.com/golang/protobuf/ptypes/empty"
"github.com/pkg/errors"
"google.golang.org/grpc"
"github.com/pulumi/pulumi/sdk/go/common/util/buildutil"
"github.com/pulumi/pulumi/sdk/go/common/util/cmdutil"
"github.com/pulumi/pulumi/sdk/go/common/util/logging"
"github.com/pulumi/pulumi/sdk/go/common/util/rpcutil"
@ -157,10 +162,109 @@ func newLanguageHost(engineAddress, tracing string) pulumirpc.LanguageRuntimeSer
}
}
// modInfo is the useful portion of the output from `go list -m -json all`
// with respect to plugin acquisition
type modInfo struct {
Path string
Version string
}
func (m *modInfo) getPlugin() (*pulumirpc.PluginDependency, error) {
if !strings.HasPrefix(m.Path, "github.com/pulumi/pulumi-") {
return nil, errors.New("module is not a pulumi provider")
}
// github.com/pulumi/pulumi-aws/sdk/... => aws
pluginPart := strings.Split(m.Path, "/")[2]
name := strings.SplitN(pluginPart, "-", 2)[1]
v, err := semver.ParseTolerant(m.Version)
if err != nil {
return nil, errors.New("module does not have semver compatible version")
}
version := m.Version
// psuedoversions are commits that don't have a corresponding tag at the specified git hash
// https://golang.org/cmd/go/#hdr-Pseudo_versions
// pulumi-aws v1.29.1-0.20200403140640-efb5e2a48a86 (first commit after 1.29.0 release)
if buildutil.IsPseudoVersion(version) {
// no prior tag means there was never a release build
if v.Major == 0 && v.Minor == 0 && v.Patch == 0 {
return nil, errors.New("invalid pseduoversion with no prior tag")
}
// patch is typically bumped from the previous tag when using pseudo version
// downgrade the patch by 1 to make sure we match a release that exists
patch := v.Patch
if patch > 0 {
patch--
}
version = fmt.Sprintf("v%v.%v.%v", v.Major, v.Minor, patch)
}
plugin := &pulumirpc.PluginDependency{
Name: name,
Version: version,
Kind: "resource",
}
return plugin, nil
}
// GetRequiredPlugins computes the complete set of anticipated plugins required by a program.
// We're lenient here as this relies on the `go list` command and the use of modules.
// If the consumer insists on using some other form of dependency management tool like
// dep or glide, the list command fails with "go list -m: not using modules"
func (host *goLanguageHost) GetRequiredPlugins(ctx context.Context,
req *pulumirpc.GetRequiredPluginsRequest) (*pulumirpc.GetRequiredPluginsResponse, error) {
return &pulumirpc.GetRequiredPluginsResponse{}, nil
logging.V(5).Infof("GetRequiredPlugins: Determining pulumi packages")
gobin, err := findExecutable("go")
if err != nil {
return nil, errors.Wrap(err, "couldn't find go binary")
}
// don't wire up stderr so non-module users don't see error output from list
cmd := exec.Command(gobin, "list", "-m", "-json", "all")
cmd.Env = os.Environ()
stdout, err := cmd.Output()
if err != nil {
// will err if the project isn't using modules
logging.V(5).Infof("GetRequiredPlugins: Error discovering plugin requirements: %s", err.Error())
return &pulumirpc.GetRequiredPluginsResponse{}, nil
}
plugins := []*pulumirpc.PluginDependency{}
dec := json.NewDecoder(bytes.NewReader(stdout))
for {
var m modInfo
if err := dec.Decode(&m); err != nil {
if err == io.EOF {
break
}
logging.V(5).Infof("GetRequiredPlugins: Error parsing list output: %s", err.Error())
return &pulumirpc.GetRequiredPluginsResponse{}, nil
}
plugin, err := m.getPlugin()
if err == nil {
logging.V(5).Infof("GetRequiredPlugins: Found plugin name: %s, version: %s", plugin.Name, plugin.Version)
plugins = append(plugins, plugin)
} else {
logging.V(5).Infof(
"GetRequiredPlugins: Ignoring dependency: %s, version: %s, error: %s",
m.Path,
m.Version,
err.Error(),
)
}
}
return &pulumirpc.GetRequiredPluginsResponse{
Plugins: plugins,
}, nil
}
// RPC endpoint for LanguageRuntimeServer::Run

View file

@ -0,0 +1,66 @@
package main
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetPlugin(t *testing.T) {
validPulumiMod := &modInfo{
Path: "github.com/pulumi/pulumi-aws/sdk",
Version: "v1.29.0",
}
validPlugin, err := validPulumiMod.getPlugin()
assert.Nil(t, err)
assert.Equal(t, validPlugin.Name, "aws")
assert.Equal(t, validPlugin.Version, "v1.29.0")
pulumiPseudoVersionModule := &modInfo{
Path: "github.com/pulumi/pulumi-aws/sdk",
Version: "v1.29.1-0.20200403140640-efb5e2a48a86",
}
pulumiPseduoVersionPlugin, err := pulumiPseudoVersionModule.getPlugin()
assert.Nil(t, err)
assert.Equal(t, pulumiPseduoVersionPlugin.Name, "aws")
assert.Equal(t, pulumiPseduoVersionPlugin.Version, "v1.29.0")
nonPulumiMod := &modInfo{
Path: "github.com/moolumi/pulumi-aws/sdk",
Version: "v1.29.0",
}
_, err = nonPulumiMod.getPlugin()
assert.NotNil(t, err)
invalidVersionModule := &modInfo{
Path: "github.com/pulumi/pulumi-aws/sdk",
Version: "42-42-42",
}
_, err = invalidVersionModule.getPlugin()
assert.NotNil(t, err)
pulumiPulumiMod := &modInfo{
Path: "github.com/pulumi/pulumi/sdk",
Version: "v1.14.0",
}
_, err = pulumiPulumiMod.getPlugin()
assert.NotNil(t, err)
betaPulumiModule := &modInfo{
Path: "github.com/pulumi/pulumi-aws/sdk",
Version: "v2.0.0-beta.1",
}
betaPulumiPlugin, err := betaPulumiModule.getPlugin()
assert.Nil(t, err)
assert.Equal(t, betaPulumiPlugin.Name, "aws")
assert.Equal(t, betaPulumiPlugin.Version, "v2.0.0-beta.1")
nonZeroPatchModule := &modInfo{
Path: "github.com/pulumi/pulumi-kubernetes/sdk",
Version: "v1.5.8",
}
nonZeroPatchPlugin, err := nonZeroPatchModule.getPlugin()
assert.Nil(t, err)
assert.Equal(t, nonZeroPatchPlugin.Name, "kubernetes")
assert.Equal(t, nonZeroPatchPlugin.Version, "v1.5.8")
}