Wait between calls to fetch update logs.

As it stands, we currently hammer the service's update logs endpoint in
a tight loop while waiting for a deployment to complete. This is not
necessary, and can indeed be deletrious to the user experience: it
appears that this may be exacerbating some mysterious 500 responses from
API gateway.

These changes add a brief sleep in the relevant loop that waits for 5
seconds if the last call produced new log entries or 15 seconds if it
did not.

Fixes #844.

pkg/backend/cloud/backend.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -574,16 +575,15 @@ type displayEvent struct {
 // waitForUpdate waits for the current update of a Pulumi program to reach a terminal state. Returns the
 // final state. "path" is the URL endpoint to poll for updates.
 func (b *cloudBackend) waitForUpdate(path string, displayOpts backend.DisplayOptions) (apitype.UpdateStatus, error) {
-	events := make(chan displayEvent)
-	done := make(chan bool)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
+	events, done := make(chan displayEvent), make(chan bool)
 	defer func() {
 		events <- displayEvent{Kind: ShutdownEvent, Payload: nil}
 		<-done
 		close(events)
 		close(done)
 	}()
-
 	go displayEvents(events, done, displayOpts)
 
 	// Events occur in sequence, filter out all the ones we have seen before in each request.
@@ -614,6 +614,16 @@ func (b *cloudBackend) waitForUpdate(path string, displayOpts backend.DisplayOpt
 		case apitype.StatusSucceeded:
 			return updateResults.Status, nil
 		}
+
+		// Wait a bit before looping again. The amount of time we wait depends on whether or not there were events: if
+		// there were no events, we wait longer than if there were events under the assumption that no new events
+		// implies a long-running but silent process in the update. In both cases, we incorporate some jitter in the
+		// wait time in order to avoid request convoys.
+		sleepTime := time.Duration(1.0+r.Float64()*4.0) * time.Second
+		if len(updateResults.Events) == 0 {
+			sleepTime += 10 * time.Second
+		}
+		time.Sleep(sleepTime)
 	}
 }