2017-04-07 08:08:33 +02:00
|
|
|
/*
|
2019-04-09 20:39:42 +02:00
|
|
|
* MinIO Cloud Storage, (C) 2017 MinIO, Inc.
|
2017-04-07 08:08:33 +02:00
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
2021-03-24 18:25:27 +01:00
|
|
|
"context"
|
2017-04-07 08:08:33 +02:00
|
|
|
"net/http"
|
2019-10-23 06:01:14 +02:00
|
|
|
"strings"
|
|
|
|
"sync"
|
2020-05-04 07:35:40 +02:00
|
|
|
"sync/atomic"
|
2017-04-07 08:08:33 +02:00
|
|
|
|
2020-04-30 20:27:19 +02:00
|
|
|
"github.com/minio/minio/cmd/logger"
|
2018-04-19 01:01:42 +02:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2017-04-07 08:08:33 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
// ConnStats - Network statistics
|
|
|
|
// Count total input/output transferred bytes during
|
|
|
|
// the server's life.
|
|
|
|
type ConnStats struct {
|
2020-05-04 07:35:40 +02:00
|
|
|
totalInputBytes uint64
|
|
|
|
totalOutputBytes uint64
|
|
|
|
s3InputBytes uint64
|
|
|
|
s3OutputBytes uint64
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Increase total input bytes
|
2019-02-15 02:53:46 +01:00
|
|
|
func (s *ConnStats) incInputBytes(n int) {
|
2020-05-04 07:35:40 +02:00
|
|
|
atomic.AddUint64(&s.totalInputBytes, uint64(n))
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Increase total output bytes
|
2019-02-15 02:53:46 +01:00
|
|
|
func (s *ConnStats) incOutputBytes(n int) {
|
2020-05-04 07:35:40 +02:00
|
|
|
atomic.AddUint64(&s.totalOutputBytes, uint64(n))
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return total input bytes
|
|
|
|
func (s *ConnStats) getTotalInputBytes() uint64 {
|
2020-05-04 07:35:40 +02:00
|
|
|
return atomic.LoadUint64(&s.totalInputBytes)
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return total output bytes
|
|
|
|
func (s *ConnStats) getTotalOutputBytes() uint64 {
|
2020-05-04 07:35:40 +02:00
|
|
|
return atomic.LoadUint64(&s.totalOutputBytes)
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// Increase outbound input bytes
|
|
|
|
func (s *ConnStats) incS3InputBytes(n int) {
|
2020-05-04 07:35:40 +02:00
|
|
|
atomic.AddUint64(&s.s3InputBytes, uint64(n))
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Increase outbound output bytes
|
|
|
|
func (s *ConnStats) incS3OutputBytes(n int) {
|
2020-05-04 07:35:40 +02:00
|
|
|
atomic.AddUint64(&s.s3OutputBytes, uint64(n))
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return outbound input bytes
|
|
|
|
func (s *ConnStats) getS3InputBytes() uint64 {
|
2020-05-04 07:35:40 +02:00
|
|
|
return atomic.LoadUint64(&s.s3InputBytes)
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return outbound output bytes
|
|
|
|
func (s *ConnStats) getS3OutputBytes() uint64 {
|
2020-05-04 07:35:40 +02:00
|
|
|
return atomic.LoadUint64(&s.s3OutputBytes)
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return connection stats (total input/output bytes and total s3 input/output bytes)
|
2017-04-21 16:15:53 +02:00
|
|
|
func (s *ConnStats) toServerConnStats() ServerConnStats {
|
|
|
|
return ServerConnStats{
|
2021-01-19 05:35:38 +01:00
|
|
|
TotalInputBytes: s.getTotalInputBytes(), // Traffic including reserved bucket
|
|
|
|
TotalOutputBytes: s.getTotalOutputBytes(), // Traffic including reserved bucket
|
|
|
|
S3InputBytes: s.getS3InputBytes(), // Traffic for client buckets
|
|
|
|
S3OutputBytes: s.getS3OutputBytes(), // Traffic for client buckets
|
2017-04-21 16:15:53 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-07 08:08:33 +02:00
|
|
|
// Prepare new ConnStats structure
|
|
|
|
func newConnStats() *ConnStats {
|
|
|
|
return &ConnStats{}
|
|
|
|
}
|
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// HTTPAPIStats holds statistics information about
|
|
|
|
// a given API in the requests.
|
|
|
|
type HTTPAPIStats struct {
|
2020-01-16 10:35:30 +01:00
|
|
|
apiStats map[string]int
|
2019-10-23 06:01:14 +02:00
|
|
|
sync.RWMutex
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// Inc increments the api stats counter.
|
|
|
|
func (stats *HTTPAPIStats) Inc(api string) {
|
|
|
|
if stats == nil {
|
|
|
|
return
|
|
|
|
}
|
2020-03-19 00:19:29 +01:00
|
|
|
stats.Lock()
|
|
|
|
defer stats.Unlock()
|
2020-01-16 10:35:30 +01:00
|
|
|
if stats.apiStats == nil {
|
|
|
|
stats.apiStats = make(map[string]int)
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
2020-03-19 00:19:29 +01:00
|
|
|
stats.apiStats[api]++
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
2017-04-07 08:08:33 +02:00
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// Dec increments the api stats counter.
|
|
|
|
func (stats *HTTPAPIStats) Dec(api string) {
|
|
|
|
if stats == nil {
|
|
|
|
return
|
|
|
|
}
|
2020-03-19 00:19:29 +01:00
|
|
|
stats.Lock()
|
|
|
|
defer stats.Unlock()
|
2020-01-16 10:35:30 +01:00
|
|
|
if val, ok := stats.apiStats[api]; ok && val > 0 {
|
|
|
|
stats.apiStats[api]--
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
|
|
|
}
|
2017-04-07 08:08:33 +02:00
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// Load returns the recorded stats.
|
|
|
|
func (stats *HTTPAPIStats) Load() map[string]int {
|
|
|
|
stats.Lock()
|
|
|
|
defer stats.Unlock()
|
2020-01-16 10:35:30 +01:00
|
|
|
var apiStats = make(map[string]int, len(stats.apiStats))
|
|
|
|
for k, v := range stats.apiStats {
|
|
|
|
apiStats[k] = v
|
|
|
|
}
|
|
|
|
return apiStats
|
2019-10-23 06:01:14 +02:00
|
|
|
}
|
2017-04-07 08:08:33 +02:00
|
|
|
|
2019-10-23 06:01:14 +02:00
|
|
|
// HTTPStats holds statistics information about
|
|
|
|
// HTTP requests made by all clients
|
|
|
|
type HTTPStats struct {
|
2021-02-20 09:21:55 +01:00
|
|
|
s3RequestsInQueue int32
|
2019-10-23 06:01:14 +02:00
|
|
|
currentS3Requests HTTPAPIStats
|
|
|
|
totalS3Requests HTTPAPIStats
|
|
|
|
totalS3Errors HTTPAPIStats
|
2021-03-24 18:25:27 +01:00
|
|
|
totalS3Canceled HTTPAPIStats
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
2021-02-20 09:21:55 +01:00
|
|
|
func (st *HTTPStats) addRequestsInQueue(i int32) {
|
|
|
|
atomic.AddInt32(&st.s3RequestsInQueue, i)
|
|
|
|
}
|
|
|
|
|
2017-04-07 08:08:33 +02:00
|
|
|
// Converts http stats into struct to be sent back to the client.
|
2019-10-23 06:01:14 +02:00
|
|
|
func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
|
2017-04-07 08:08:33 +02:00
|
|
|
serverStats := ServerHTTPStats{}
|
2021-02-20 09:21:55 +01:00
|
|
|
serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
|
2019-10-23 06:01:14 +02:00
|
|
|
serverStats.CurrentS3Requests = ServerHTTPAPIStats{
|
|
|
|
APIStats: st.currentS3Requests.Load(),
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
2019-10-23 06:01:14 +02:00
|
|
|
serverStats.TotalS3Requests = ServerHTTPAPIStats{
|
|
|
|
APIStats: st.totalS3Requests.Load(),
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
2019-10-23 06:01:14 +02:00
|
|
|
serverStats.TotalS3Errors = ServerHTTPAPIStats{
|
|
|
|
APIStats: st.totalS3Errors.Load(),
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
2021-03-24 18:25:27 +01:00
|
|
|
serverStats.TotalS3Canceled = ServerHTTPAPIStats{
|
|
|
|
APIStats: st.totalS3Canceled.Load(),
|
|
|
|
}
|
2017-04-07 08:08:33 +02:00
|
|
|
return serverStats
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update statistics from http request and response data
|
2020-12-11 08:02:25 +01:00
|
|
|
func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.ResponseWriter) {
|
2017-04-07 08:08:33 +02:00
|
|
|
// A successful request has a 2xx response code
|
2021-01-19 05:35:38 +01:00
|
|
|
successReq := w.StatusCode >= 200 && w.StatusCode < 300
|
2019-10-23 06:01:14 +02:00
|
|
|
|
2021-01-19 05:35:38 +01:00
|
|
|
if !strings.HasSuffix(r.URL.Path, prometheusMetricsPathLegacy) ||
|
|
|
|
!strings.HasSuffix(r.URL.Path, prometheusMetricsV2ClusterPath) ||
|
|
|
|
!strings.HasSuffix(r.URL.Path, prometheusMetricsV2NodePath) {
|
2019-10-23 06:01:14 +02:00
|
|
|
st.totalS3Requests.Inc(api)
|
2020-04-30 20:27:19 +02:00
|
|
|
if !successReq && w.StatusCode != 0 {
|
2019-10-23 06:01:14 +02:00
|
|
|
st.totalS3Errors.Inc(api)
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
2021-03-24 18:25:27 +01:00
|
|
|
select {
|
|
|
|
case <-r.Context().Done():
|
|
|
|
if err := r.Context().Err(); err == context.Canceled {
|
|
|
|
st.totalS3Canceled.Inc(api)
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
}
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
2019-10-23 06:01:14 +02:00
|
|
|
|
2020-12-11 08:02:25 +01:00
|
|
|
// Increment the prometheus http request response histogram with appropriate label
|
|
|
|
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
|
2017-04-07 08:08:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Prepare new HTTPStats structure
|
|
|
|
func newHTTPStats() *HTTPStats {
|
|
|
|
return &HTTPStats{}
|
|
|
|
}
|