Fix python package versions

Our logic for converting npm style versions to PEP-440 style versions
was not correct in some cases. This change fixes this.

As part of this change we no longer produce a NPM version that would
be just X.Y.Z-dev, instead for development versions we always include
both the timestamp of the commit and the commit hash.

Instead of trying to use a bunch of sed logic to do our conversions,
we now have a small go program that uses a newly added library in
pkg/util. A side effect of this is that we can more easily write tests
to ensure the conversion works as expected.

Fixes #1243

pkg/util/buildutil/semver.go

@@ -0,0 +1,81 @@
+package buildutil
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+
+	"github.com/pkg/errors"
+	"github.com/pulumi/pulumi/pkg/util/contract"
+)
+
+var (
+	releaseVersionRegex = regexp.MustCompile(
+		`^v(?P<version>\d+\.\d+\.\d+)(-(?P<time>\d+)-(?P<gitInfo>g[a-z0-9]+))?(?P<dirty>-dirty)?$`)
+	rcVersionRegex = regexp.MustCompile(
+		`^v(?P<version>\d+\.\d+\.\d+)-rc(?P<rcN>\d+)(-(?P<time>\d+)-(?P<gitInfo>g[a-z0-9]+))?(?P<dirty>-dirty)?$`)
+	devVersionRegex = regexp.MustCompile(
+		`^v(?P<version>\d+\.\d+\.\d+)-dev-(?P<time>\d+)-(?P<gitInfo>g[a-z0-9]+)(?P<dirty>-dirty)?$`)
+)
+
+// PyPiVersionFromNpmVersion returns a PEP-440 compliant version for a given semver version. This method does not
+// support all possible semver strings, but instead just supports versions that we generate for our node packages.
+func PyPiVersionFromNpmVersion(s string) (string, error) {
+	var b bytes.Buffer
+
+	if releaseVersionRegex.MatchString(s) {
+		capMap := captureToMap(releaseVersionRegex, s)
+		mustFprintf(&b, "%s", capMap["version"])
+		writePostBuildAndDirtyInfoToReleaseVersion(&b, capMap)
+		return b.String(), nil
+
+	} else if rcVersionRegex.MatchString(s) {
+		capMap := captureToMap(rcVersionRegex, s)
+		mustFprintf(&b, "%src%s", capMap["version"], capMap["rcN"])
+		writePostBuildAndDirtyInfoToReleaseVersion(&b, capMap)
+		return b.String(), nil
+	} else if devVersionRegex.MatchString(s) {
+		capMap := captureToMap(devVersionRegex, s)
+		mustFprintf(&b, "%s.dev%s+%s", capMap["version"], capMap["time"], capMap["gitInfo"])
+		if capMap["dirty"] != "" {
+			mustFprintf(&b, ".dirty")
+		}
+		return b.String(), nil
+	}
+
+	return "", errors.Errorf("can not parse version string '%s'", s)
+}
+
+func captureToMap(r *regexp.Regexp, s string) map[string]string {
+	matches := r.FindStringSubmatch(s)
+	capMap := make(map[string]string)
+	for i, name := range r.SubexpNames() {
+		if name != "" {
+			capMap[name] = matches[i]
+		}
+	}
+
+	return capMap
+}
+
+// While the version string for dev builds always contain timestamp and commit information, release and release
+// release candidate builds do not. In the case where we do have this information, it is for a build newer than
+// the actual release build, and we'll use the PEP-440 .post notation to show this. We also handle adding the dirty
+// tag in the local version if we need it.
+func writePostBuildAndDirtyInfoToReleaseVersion(w io.Writer, capMap map[string]string) {
+	if capMap["time"] != "" {
+		mustFprintf(w, ".post%s", capMap["time"])
+		mustFprintf(w, "+%s", capMap["gitInfo"])
+		if capMap["dirty"] != "" {
+			mustFprintf(w, ".dirty")
+		}
+	} else if capMap["dirty"] != "" {
+		mustFprintf(w, "+dirty")
+	}
+}
+
+func mustFprintf(w io.Writer, format string, a ...interface{}) {
+	_, err := fmt.Fprintf(w, format, a...)
+	contract.AssertNoError(err)
+}

pkg/util/buildutil/semver_test.go

@@ -0,0 +1,27 @@
+package buildutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVersions(t *testing.T) {
+	cases := map[string]string{
+		"v0.12.0":                                "0.12.0",
+		"v0.12.0-dirty":                          "0.12.0+dirty",
+		"v0.12.0-1524606809-gf2f1178b":           "0.12.0.post1524606809+gf2f1178b",
+		"v0.12.0-1524606809-gf2f1178b-dirty":     "0.12.0.post1524606809+gf2f1178b.dirty",
+		"v0.12.0-rc1":                            "0.12.0rc1",
+		"v0.12.0-rc1-1524606809-gf2f1178b":       "0.12.0rc1.post1524606809+gf2f1178b",
+		"v0.12.0-rc1-1524606809-gf2f1178b-dirty": "0.12.0rc1.post1524606809+gf2f1178b.dirty",
+		"v0.12.1-dev-1524606809-gf2f1178b":       "0.12.1.dev1524606809+gf2f1178b",
+		"v0.12.1-dev-1524606809-gf2f1178b-dirty": "0.12.1.dev1524606809+gf2f1178b.dirty",
+	}
+
+	for ver, expected := range cases {
+		p, err := PyPiVersionFromNpmVersion(ver)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, p, "failed parsing '%s'", ver)
+	}
+}

scripts/get-py-version

@@ -2,20 +2,6 @@
 set -o nounset -o errexit -o pipefail
 
 SCRIPT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+NPM_VERSION="$($SCRIPT_ROOT/get-version "$@")"
 
-# To generate a Python version, we will need to mangle it
-# slightly.  Namely, we must perform the following translations:
-#
-#     1) Skip the leading "v" (i.e., "1.3.11", not "v1.3.11").
-#     2) Change "-dev-<xyz>" into an alpha release "a<xyz>".
-#     3) Change "-rc-<xyz>" into a release candidate "rc<xyz>".
-#     4) Change "-<commitish><dirty>" into a local version label;
-#        e.g. "+37bc2f9-dirty" rather than "-37bc2f9-dirty".
-#
-# This ensures conformance with PEP440:
-# https://www.python.org/dev/peps/pep-0440/#version-scheme.
-echo $($SCRIPT_ROOT/get-version "$@") | \
-    sed 's/v//'      | \
-    sed 's/-dev-/a/' | \
-    sed 's/-rc-/rc/' | \
-    sed 's/-/+/'
+go run "${SCRIPT_ROOT}/get-py-version.go" "${NPM_VERSION}"

scripts/get-py-version.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pulumi/pulumi/pkg/util/buildutil"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "error: need exactly one argument\n")
+		os.Exit(-1)
+	}
+
+	p, err := buildutil.PyPiVersionFromNpmVersion(os.Args[1])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %s\n", err)
+		os.Exit(-1)
+	}
+
+	fmt.Println(p)
+}

scripts/get-version

@@ -12,14 +12,17 @@ if ! git diff-files --quiet; then
     DIRTY_TAG="-dirty"
 fi
 
-# If the commit in question has a tag applied to it directly, use it.
+# If we have an exact tag, and it is not a -dev tag, just use it.
 if git describe --tags --exact-match "${COMMITISH}" >/dev/null 2>&1; then
-    echo "$(git describe --tags --exact-match "${COMMITISH}")${DIRTY_TAG}"
-    exit 0
+    TAG="$(git describe --tags --exact-match "${COMMITISH}")"
+    if [[ ! "${TAG}" =~ -dev$ ]]; then
+        echo "$(git describe --tags --exact-match "${COMMITISH}")${DIRTY_TAG}"
+        exit 0
+    fi
 fi
 
-# Otherwise we don't have an exact tag, so produce one that is the
-# base tag, plus a timestamp and commit hash. We use the timestamp of
+# Otherwise we want to include some additional information. To the
+# base tag we add a timestamp and commit hash. We use the timestamp of
 # the commit itself, not the date it was authored (so it will change
 # when someone rebases a PR into master, for example).
 echo "$(git describe --tags --abbrev=0 ${COMMITISH})-$(git show -s --format='%ct-g%h' ${COMMITISH})${DIRTY_TAG}"