Make credentials.json file writes concurrency safe (#5857)

Running `pulumi` operations in parallel could occasionally result in truncating the `~/.pulumi/credentials.json` file and reading that truncated file from another process before the content could be written.

Instead, use `os.Rename` to atomically replace the file contents.

Concurrent `pulumi` operations could still compete for who gets to write the file first, and could lead to surprising results in some extreme cases.   But we should not see the corrupted file contents any longer.

Fixes #3877.

CHANGELOG.md

@@ -2,7 +2,10 @@ CHANGELOG
 =========
 
 ## HEAD (Unreleased)
-_(none)_
+
+- [sdk/python] Address potential issues when running multiple `pulumi` processes concurrently.
+  [#5857](https://github.com/pulumi/pulumi/pull/5857)
+
 
 ## 2.15.0 (2020-12-02)
 

sdk/go/common/workspace/creds.go

@@ -18,11 +18,13 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
 
+	"github.com/pulumi/pulumi/sdk/v2/go/common/util/contract"
 	"github.com/pulumi/pulumi/sdk/v2/go/common/util/logging"
 )
 
@@ -213,5 +215,26 @@ func StoreCredentials(creds Credentials) error {
 	if err != nil {
 		return errors.Wrapf(err, "marshalling credentials object")
 	}
-	return ioutil.WriteFile(credsFile, raw, 0600)
+
+	// Use a temporary file and atomic os.Rename to ensure the file contents are
+	// updated atomically to ensure concurrent `pulumi` CLI operations are safe.
+	tempCredsFile, err := ioutil.TempFile(path.Dir(credsFile), "credentials-*.json")
+	if err != nil {
+		return err
+	}
+	_, err = tempCredsFile.Write(raw)
+	if err != nil {
+		return err
+	}
+	err = tempCredsFile.Close()
+	if err != nil {
+		return err
+	}
+	err = os.Rename(tempCredsFile.Name(), credsFile)
+	if err != nil {
+		contract.IgnoreError(os.Remove(tempCredsFile.Name()))
+		return err
+	}
+
+	return nil
 }

sdk/go/common/workspace/creds_test.go

@@ -0,0 +1,30 @@
+package workspace
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConcurrentCredentialsWrites(t *testing.T) {
+	creds, err := GetStoredCredentials()
+	assert.NoError(t, err)
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			err := StoreCredentials(creds)
+			assert.NoError(t, err)
+		}()
+		go func() {
+			defer wg.Done()
+			_, err := GetStoredCredentials()
+			assert.NoError(t, err)
+		}()
+	}
+	wg.Wait()
+}