Add Profiler Admin API (#6463)

Two handlers are added to admin API to enable profiling and disable
profiling of a server in a standalone mode, or all nodes in the
distributed mode.

/minio/admin/profiling/start/{cpu,block,mem}:
  - Start profiling and return starting JSON results, e.g. one
    node is offline.

/minio/admin/profiling/download:
  - Stop the on-going profiling task
  - Stream a zip file which contains all profiling files that can
    be later inspected by go tool pprof

cmd/admin-handlers.go

@@ -17,13 +17,16 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -275,6 +278,121 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// StartProfilingResult contains the status of the starting
+// profiling action in a given server
+type StartProfilingResult struct {
+	NodeName string `json:"nodeName"`
+	Success  bool   `json:"success"`
+	Error    string `json:"error"`
+}
+
+// StartProfilingHandler - POST /minio/admin/v1/profiling/start/{profiler}
+// ----------
+// Enable profiling information
+func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	profiler := vars["profiler"]
+
+	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
+
+	// Call StartProfiling function on all nodes and save results
+	wg := sync.WaitGroup{}
+	for i, peer := range globalAdminPeers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			result := StartProfilingResult{NodeName: peer.addr}
+			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
+				result.Error = err.Error()
+				return
+			}
+			result.Success = true
+			startProfilingResult[idx] = result
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Create JSON result and send it to the client
+	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
+	if err != nil {
+		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+}
+
+// dummyFileInfo represents a dummy representation of a profile data file
+// present only in memory, it helps to generate the zip stream.
+type dummyFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+	sys     interface{}
+}
+
+func (f dummyFileInfo) Name() string       { return f.name }
+func (f dummyFileInfo) Size() int64        { return f.size }
+func (f dummyFileInfo) Mode() os.FileMode  { return f.mode }
+func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
+func (f dummyFileInfo) IsDir() bool        { return f.isDir }
+func (f dummyFileInfo) Sys() interface{}   { return f.sys }
+
+// DownloadProfilingHandler - POST /minio/admin/v1/profiling/download
+// ----------
+// Download profiling information of all nodes in a zip format
+func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Return 200 OK
+	w.WriteHeader(http.StatusOK)
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+
+	for i, peer := range globalAdminPeers {
+		// Get profiling data from a node
+		data, err := peer.cmdRunner.DownloadProfilingData()
+		if err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
+			continue
+		}
+
+		// Send profiling data to zip as file
+		header, err := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%d", i),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: time.Now().UTC(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if err != nil {
+			continue
+		}
+		writer, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			continue
+		}
+		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
+			return
+		}
+	}
+}
+
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	hs madmin.HealOpts, clientToken string, forceStart bool,

cmd/admin-router.go

@@ -60,6 +60,10 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start/{profiler}").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler))
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+
 	/// Config operations
 
 	// Update credentials

cmd/admin-rpc-client.go

@@ -68,6 +68,22 @@ func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
 	return reply, err
 }
 
+// StartProfiling - starts profiling in the remote server.
+func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - returns profiling data of the remote server.
+func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
 // NewAdminRPCClient - returns new admin RPC client.
 func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 	scheme := "http"
@@ -112,6 +128,8 @@ type adminCmdRunner interface {
 	ReInitFormat(dryRun bool) error
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
+	StartProfiling(string) error
+	DownloadProfilingData() ([]byte, error)
 }
 
 // adminPeer - represents an entity that implements admin API RPCs.

cmd/admin-rpc-server.go

@@ -52,6 +52,23 @@ func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoDa
 	return err
 }
 
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - starts profiling of this server
+func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	return receiver.local.StartProfiling(args.Profiler)
+}
+
+// DownloadProfilingData - stops and returns profiling data of this server
+func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.DownloadProfilingData()
+	return
+}
+
 // GetConfig - returns the config.json of this server.
 func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
 	*reply, err = receiver.local.GetConfig()

cmd/api-errors.go

@@ -1727,7 +1727,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	return errorCodeResponse[code]
+	if apiErr, ok := errorCodeResponse[code]; ok {
+		return apiErr
+	}
+	return errorCodeResponse[ErrInternalError]
 }
 
 // getErrorResponse gets in standard error and resource value and

cmd/common-main.go

@@ -98,7 +98,9 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 func handleCommonEnvVars() {
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")

cmd/local-admin-client.go

@@ -19,7 +19,11 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
+
+	"io/ioutil"
 )
 
 // localAdminClient - represents admin operation to be executed locally.
@@ -80,3 +84,40 @@ func (lc localAdminClient) GetConfig() ([]byte, error) {
 
 	return json.Marshal(globalServerConfig)
 }
+
+// StartProfiling - starts profiling on the local server.
+func (lc localAdminClient) StartProfiling(profiler string) error {
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+	prof, err := startProfiler(profiler, "")
+	if err != nil {
+		return err
+	}
+	globalProfiler = prof
+	return nil
+}
+
+// DownloadProfilingData - stops and returns profiling data of the local server.
+func (lc localAdminClient) DownloadProfilingData() ([]byte, error) {
+	if globalProfiler == nil {
+		return nil, errors.New("profiler not enabled")
+	}
+
+	profilerPath := globalProfiler.Path()
+
+	// Stop the profiler
+	globalProfiler.Stop()
+
+	profilerFile, err := os.Open(profilerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadAll(profilerFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}

cmd/utils.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -30,6 +31,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -182,26 +184,66 @@ func contains(slice interface{}, elem interface{}) bool {
 	return false
 }
 
+// profilerWrapper is created becauses pkg/profiler doesn't
+// provide any API to calculate the profiler file path in the
+// disk since the name of this latter is randomly generated.
+type profilerWrapper struct {
+	stopFn func()
+	pathFn func() string
+}
+
+func (p profilerWrapper) Stop() {
+	p.stopFn()
+}
+
+func (p profilerWrapper) Path() string {
+	return p.pathFn()
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
-func startProfiler(profiler string) interface {
+func startProfiler(profilerType, dirPath string) (interface {
 	Stop()
-} {
-	// Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
-	switch profiler {
-	case "cpu":
-		return profile.Start(profile.CPUProfile, profile.NoShutdownHook)
-	case "mem":
-		return profile.Start(profile.MemProfile, profile.NoShutdownHook)
-	case "block":
-		return profile.Start(profile.BlockProfile, profile.NoShutdownHook)
-	default:
-		return nil
+	Path() string
+}, error) {
+
+	var err error
+	if dirPath == "" {
+		dirPath, err = ioutil.TempDir("", "profile")
+		if err != nil {
+			return nil, err
+		}
 	}
+
+	var profiler interface {
+		Stop()
+	}
+
+	// Enable profiler, supported types are [cpu, mem, block].
+	switch profilerType {
+	case "cpu":
+		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "mem":
+		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "block":
+		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	default:
+		return nil, errors.New("profiler type unknown")
+	}
+
+	return &profilerWrapper{
+		stopFn: profiler.Stop,
+		pathFn: func() string {
+			return filepath.Join(dirPath, profilerType+".pprof")
+		},
+	}, nil
 }
 
 // Global profiler to be used by service go-routine.
 var globalProfiler interface {
+	// Stop the profiler
 	Stop()
+	// Return the path of the profiling file
+	Path() string
 }
 
 // dump the request into a string in JSON format.

cmd/utils_test.go

@@ -228,8 +228,9 @@ func TestURL2BucketObjectName(t *testing.T) {
 
 // Add tests for starting and stopping different profilers.
 func TestStartProfiler(t *testing.T) {
-	if startProfiler("") != nil {
-		t.Fatal("Expected nil, but non-nil value returned for invalid profiler.")
+	_, err := startProfiler("", "")
+	if err == nil {
+		t.Fatal("Expected a non nil error, but nil error returned for invalid profiler.")
 	}
 }
 

pkg/madmin/API.md

@@ -39,8 +39,8 @@ func main() {
 | Service operations         | Info operations  | Healing operations                    | Config operations         | Misc                                |
 |:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|
 | [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`SetCredentials`](#SetCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | | | [`SetConfig`](#SetConfig) | |
-| | |            | [`GetConfigKeys`](#GetConfigKeys) |                                     |
+| [`ServiceSendAction`](#ServiceSendAction) | | | [`SetConfig`](#SetConfig) | [`StartProfiling`](#StartProfiling) |
+| | |            | [`GetConfigKeys`](#GetConfigKeys) | [`DownloadProfilingData`](#DownloadProfilingData) |
 | | |            | [`SetConfigKeys`](#SetConfigKeys) |                                     |
 
 
@@ -385,3 +385,56 @@ __Example__
     log.Println("New credentials successfully set.")
 
 ```
+
+<a name="StartProfiling"></a>
+### StartProfiling(profiler string) error
+Ask all nodes to start profiling using the specified profiler mode
+
+__Example__
+
+``` go
+    startProfilingResults, err = madmClnt.StartProfiling("cpu")
+    if err != nil {
+            log.Fatalln(err)
+    }
+    for _, result := range startProfilingResults {
+        if !result.Success {
+            log.Printf("Unable to start profiling on node `%s`, reason = `%s`\n", result.NodeName, result.Error)
+        } else {
+            log.Printf("Profiling successfully started on node `%s`\n", result.NodeName)
+        }
+    }
+
+```
+
+<a name="DownloadProfilingData"></a>
+### DownloadProfilingData() ([]byte, error)
+Download profiling data of all nodes in a zip format.
+
+__Example__
+
+``` go
+    profilingData, err := madmClnt.DownloadProfilingData()
+    if err != nil {
+            log.Fatalln(err)
+    }
+
+    profilingFile, err := os.Create("/tmp/profiling-data.zip")
+    if err != nil {
+            log.Fatal(err)
+    }
+
+    if _, err := io.Copy(profilingFile, profilingData); err != nil {
+            log.Fatal(err)
+    }
+
+    if err := profilingFile.Close(); err != nil {
+            log.Fatal(err)
+    }
+
+    if err := profilingData.Close(); err != nil {
+            log.Fatal(err)
+    }
+
+    log.Println("Profiling data successfully downloaded.")
+```

pkg/madmin/examples/profiling.go

@@ -0,0 +1,89 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"time"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	profiler := madmin.ProfilerCPU
+	log.Println("Starting " + profiler + " profiling..")
+
+	startResults, err := madmClnt.StartProfiling(profiler)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	for _, result := range startResults {
+		if !result.Success {
+			log.Printf("Unable to start profiling on node `%s`, reason = `%s`\n", result.NodeName, result.Error)
+			continue
+		}
+		log.Printf("Profiling successfully started on node `%s`\n", result.NodeName)
+	}
+
+	sleep := time.Duration(10)
+	time.Sleep(time.Second * sleep)
+
+	log.Println("Stopping profiling..")
+
+	profilingData, err := madmClnt.DownloadProfilingData()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	profilingFile, err := os.Create("/tmp/profiling-" + string(profiler) + ".zip")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := io.Copy(profilingFile, profilingData); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := profilingFile.Close(); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := profilingData.Close(); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("Profiling files " + profilingFile.Name() + " successfully downloaded.")
+}

pkg/madmin/profiling-commands.go

@@ -0,0 +1,104 @@
+/*
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+// ProfilerType represents the profiler type
+// passed to the profiler subsystem, currently
+// it can be only "cpu", "mem" or "block"
+type ProfilerType string
+
+const (
+	// ProfilerCPU represents CPU profiler type
+	ProfilerCPU = ProfilerType("cpu")
+	// ProfilerMEM represents MEM profiler type
+	ProfilerMEM = ProfilerType("mem")
+	// ProfilerBlock represents Block profiler type
+	ProfilerBlock = ProfilerType("block")
+)
+
+// StartProfilingResult holds the result of starting
+// profiler result in a given node.
+type StartProfilingResult struct {
+	NodeName string `json:"nodeName"`
+	Success  bool   `json:"success"`
+	Error    string `json:"error"`
+}
+
+// StartProfiling makes an admin call to remotely start profiling on a standalone
+// server or the whole cluster in  case of a distributed setup.
+func (adm *AdminClient) StartProfiling(profiler ProfilerType) ([]StartProfilingResult, error) {
+	path := fmt.Sprintf("/v1/profiling/start/%s", profiler)
+	resp, err := adm.executeMethod("POST", requestData{
+		relPath: path,
+	})
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	jsonResult, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var startResults []StartProfilingResult
+	err = json.Unmarshal(jsonResult, &startResults)
+	if err != nil {
+		return nil, err
+	}
+
+	return startResults, nil
+}
+
+// DownloadProfilingData makes an admin call to download profiling data of a standalone
+// server or of the whole cluster in  case of a distributed setup.
+func (adm *AdminClient) DownloadProfilingData() (io.ReadCloser, error) {
+	path := fmt.Sprintf("/v1/profiling/download")
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath: path,
+	})
+
+	if err != nil {
+		closeResponse(resp)
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		closeResponse(resp)
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	if resp.Body == nil {
+		return nil, errors.New("body is nil")
+	}
+
+	return resp.Body, nil
+}

vendor/github.com/pkg/profile/README.md

@@ -3,6 +3,9 @@ profile
 
 Simple profiling support package for Go
 
+[![Build Status](https://travis-ci.org/pkg/profile.svg?branch=master)](https://travis-ci.org/pkg/profile) [![GoDoc](http://godoc.org/github.com/pkg/profile?status.svg)](http://godoc.org/github.com/pkg/profile)
+
+
 installation
 ------------
 
@@ -42,3 +45,10 @@ func main() {
 Several convenience package level values are provided for cpu, memory, and block (contention) profiling.
 
 For more complex options, consult the [documentation](http://godoc.org/github.com/pkg/profile).
+
+contributing
+------------
+
+We welcome pull requests, bug fixes and issue reports.
+
+Before proposing a change, please discuss it first by raising an issue.

vendor/github.com/pkg/profile/mutex.go

@@ -0,0 +1,13 @@
+// +build go1.8
+
+package profile
+
+import "runtime"
+
+func enableMutexProfile() {
+	runtime.SetMutexProfileFraction(1)
+}
+
+func disableMutexProfile() {
+	runtime.SetMutexProfileFraction(0)
+}

vendor/github.com/pkg/profile/mutex17.go

@@ -0,0 +1,9 @@
+// +build !go1.8
+
+package profile
+
+// mock mutex support for Go 1.7 and earlier.
+
+func enableMutexProfile() {}
+
+func disableMutexProfile() {}

vendor/github.com/pkg/profile/profile.go

@@ -13,16 +13,16 @@ import (
 	"sync/atomic"
 )
 
-// started counts the number of times Start has been called
-var started uint32
-
 const (
 	cpuMode = iota
 	memMode
+	mutexMode
 	blockMode
+	traceMode
 )
 
-type profile struct {
+// Profile represents an active profiling session.
+type Profile struct {
 	// quiet suppresses informational messages during profiling.
 	quiet bool
 
@@ -40,8 +40,8 @@ type profile struct {
 	// memProfileRate holds the rate for the memory profile.
 	memProfileRate int
 
-	// closers holds the cleanup functions that run after each profile
-	closers []func()
+	// closer holds a cleanup function that run after each profile
+	closer func()
 
 	// stopped records if a call to profile.Stop has been made
 	stopped uint32
@@ -52,68 +52,81 @@ type profile struct {
 // Programs with more sophisticated signal handling should set
 // this to true and ensure the Stop() function returned from Start()
 // is called during shutdown.
-func NoShutdownHook(p *profile) { p.noShutdownHook = true }
+func NoShutdownHook(p *Profile) { p.noShutdownHook = true }
 
 // Quiet suppresses informational messages during profiling.
-func Quiet(p *profile) { p.quiet = true }
+func Quiet(p *Profile) { p.quiet = true }
 
-// CPUProfile controls if cpu profiling will be enabled. It disables any previous profiling settings.
-func CPUProfile(p *profile) { p.mode = cpuMode }
+// CPUProfile enables cpu profiling.
+// It disables any previous profiling settings.
+func CPUProfile(p *Profile) { p.mode = cpuMode }
 
 // DefaultMemProfileRate is the default memory profiling rate.
 // See also http://golang.org/pkg/runtime/#pkg-variables
 const DefaultMemProfileRate = 4096
 
-// MemProfile controls if memory profiling will be enabled. It disables any previous profiling settings.
-func MemProfile(p *profile) {
+// MemProfile enables memory profiling.
+// It disables any previous profiling settings.
+func MemProfile(p *Profile) {
 	p.memProfileRate = DefaultMemProfileRate
 	p.mode = memMode
 }
 
-// MemProfileRate controls if memory profiling will be enabled. Additionally, it takes a parameter which
-// allows the setting of the memory profile rate.
-func MemProfileRate(rate int) func(*profile) {
-	return func(p *profile) {
+// MemProfileRate enables memory profiling at the preferred rate.
+// It disables any previous profiling settings.
+func MemProfileRate(rate int) func(*Profile) {
+	return func(p *Profile) {
 		p.memProfileRate = rate
 		p.mode = memMode
 	}
 }
 
-// BlockProfile controls if block (contention) profiling will be enabled. It disables any previous profiling settings.
-func BlockProfile(p *profile) { p.mode = blockMode }
+// MutexProfile enables mutex profiling.
+// It disables any previous profiling settings.
+//
+// Mutex profiling is a no-op before go1.8.
+func MutexProfile(p *Profile) { p.mode = mutexMode }
+
+// BlockProfile enables block (contention) profiling.
+// It disables any previous profiling settings.
+func BlockProfile(p *Profile) { p.mode = blockMode }
+
+// Trace profile controls if execution tracing will be enabled. It disables any previous profiling settings.
+func TraceProfile(p *Profile) { p.mode = traceMode }
 
 // ProfilePath controls the base path where various profiling
 // files are written. If blank, the base path will be generated
 // by ioutil.TempDir.
-func ProfilePath(path string) func(*profile) {
-	return func(p *profile) {
+func ProfilePath(path string) func(*Profile) {
+	return func(p *Profile) {
 		p.path = path
 	}
 }
 
 // Stop stops the profile and flushes any unwritten data.
-func (p *profile) Stop() {
+func (p *Profile) Stop() {
 	if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) {
 		// someone has already called close
 		return
 	}
-	for _, c := range p.closers {
-		c()
-	}
+	p.closer()
+	atomic.StoreUint32(&started, 0)
 }
 
+// started is non zero if a profile is running.
+var started uint32
+
 // Start starts a new profiling session.
 // The caller should call the Stop method on the value returned
-// to cleanly stop profiling. Start can only be called once
-// per program execution.
-func Start(options ...func(*profile)) interface {
+// to cleanly stop profiling.
+func Start(options ...func(*Profile)) interface {
 	Stop()
 } {
 	if !atomic.CompareAndSwapUint32(&started, 0, 1) {
 		log.Fatal("profile: Start() already called")
 	}
 
-	var prof profile
+	var prof Profile
 	for _, option := range options {
 		option(&prof)
 	}
@@ -129,6 +142,12 @@ func Start(options ...func(*profile)) interface {
 		log.Fatalf("profile: could not create initial output directory: %v", err)
 	}
 
+	logf := func(format string, args ...interface{}) {
+		if !prof.quiet {
+			log.Printf(format, args...)
+		}
+	}
+
 	switch prof.mode {
 	case cpuMode:
 		fn := filepath.Join(path, "cpu.pprof")
@@ -136,14 +155,13 @@ func Start(options ...func(*profile)) interface {
 		if err != nil {
 			log.Fatalf("profile: could not create cpu profile %q: %v", fn, err)
 		}
-		if !prof.quiet {
-			log.Printf("profile: cpu profiling enabled, %s", fn)
-		}
+		logf("profile: cpu profiling enabled, %s", fn)
 		pprof.StartCPUProfile(f)
-		prof.closers = append(prof.closers, func() {
+		prof.closer = func() {
 			pprof.StopCPUProfile()
 			f.Close()
-		})
+			logf("profile: cpu profiling disabled, %s", fn)
+		}
 
 	case memMode:
 		fn := filepath.Join(path, "mem.pprof")
@@ -153,14 +171,30 @@ func Start(options ...func(*profile)) interface {
 		}
 		old := runtime.MemProfileRate
 		runtime.MemProfileRate = prof.memProfileRate
-		if !prof.quiet {
-			log.Printf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, fn)
-		}
-		prof.closers = append(prof.closers, func() {
+		logf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, fn)
+		prof.closer = func() {
 			pprof.Lookup("heap").WriteTo(f, 0)
 			f.Close()
 			runtime.MemProfileRate = old
-		})
+			logf("profile: memory profiling disabled, %s", fn)
+		}
+
+	case mutexMode:
+		fn := filepath.Join(path, "mutex.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create mutex profile %q: %v", fn, err)
+		}
+		enableMutexProfile()
+		logf("profile: mutex profiling enabled, %s", fn)
+		prof.closer = func() {
+			if mp := pprof.Lookup("mutex"); mp != nil {
+				mp.WriteTo(f, 0)
+			}
+			f.Close()
+			disableMutexProfile()
+			logf("profile: mutex profiling disabled, %s", fn)
+		}
 
 	case blockMode:
 		fn := filepath.Join(path, "block.pprof")
@@ -169,14 +203,28 @@ func Start(options ...func(*profile)) interface {
 			log.Fatalf("profile: could not create block profile %q: %v", fn, err)
 		}
 		runtime.SetBlockProfileRate(1)
-		if !prof.quiet {
-			log.Printf("profile: block profiling enabled, %s", fn)
-		}
-		prof.closers = append(prof.closers, func() {
+		logf("profile: block profiling enabled, %s", fn)
+		prof.closer = func() {
 			pprof.Lookup("block").WriteTo(f, 0)
 			f.Close()
 			runtime.SetBlockProfileRate(0)
-		})
+			logf("profile: block profiling disabled, %s", fn)
+		}
+
+	case traceMode:
+		fn := filepath.Join(path, "trace.out")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create trace output file %q: %v", fn, err)
+		}
+		if err := startTrace(f); err != nil {
+			log.Fatalf("profile: could not start trace: %v", err)
+		}
+		logf("profile: trace enabled, %s", fn)
+		prof.closer = func() {
+			stopTrace()
+			logf("profile: trace disabled, %s", fn)
+		}
 	}
 
 	if !prof.noShutdownHook {

vendor/github.com/pkg/profile/trace.go

@@ -0,0 +1,8 @@
+// +build go1.7
+
+package profile
+
+import "runtime/trace"
+
+var startTrace = trace.Start
+var stopTrace = trace.Stop

vendor/github.com/pkg/profile/trace16.go

@@ -0,0 +1,10 @@
+// +build !go1.7
+
+package profile
+
+import "io"
+
+// mock trace support for Go 1.6 and earlier.
+
+func startTrace(w io.Writer) error { return nil }
+func stopTrace()                   {}

vendor/github.com/pkg/profile/wercker.yml

@@ -1 +0,0 @@
-box: wercker/golang

vendor/vendor.json

@@ -765,9 +765,10 @@
 			"revisionTime": "2017-12-16T07:03:16Z"
 		},
 		{
+			"checksumSHA1": "C3yiSMdTQxSY3xqKJzMV9T+KnIc=",
 			"path": "github.com/pkg/profile",
-			"revision": "c78aac22bd43883fd2817833b982153dcac17b3b",
-			"revisionTime": "2016-05-18T16:56:57+10:00"
+			"revision": "057bc52a47ec3c79498dda63f4a6f8298725e976",
+			"revisionTime": "2018-08-09T11:22:05Z"
 		},
 		{
 			"checksumSHA1": "3NL4uu8RZhI2d+/SEmVOHPT390c=",